[Feature, UE4] Configurable translucency in depth and semantic + instance segmentation

CHANGELOG.md

@@ -35,6 +35,7 @@
  * Set to default Visual Studio 2022 in Windows
  * Added env CARLA_CACHE_DIR to be able to set CARLA CACHE location
  * Support of masked materials in instance segmentation, resulting in fine-grained annotations on e.g. leaves or fences (as in semantic segmentation)
+ * Added API function `world.set_annotations_traverse_translucency` and implemented functionality to configure, whether depth and semantic + instance segmentation traverse translucent materials or not.
 
 ## CARLA 0.9.15
 

Docs/python_api.md

@@ -3837,6 +3837,10 @@ Retrieves an object containing weather parameters currently active in the simula
     - **Setter:** _[carla.World.set_weather](#carla.World.set_weather)_  
 
 ##### Setters
+- <a name="carla.World.set_annotations_traverse_translucency"></a>**<font color="#7fb800">set_annotations_traverse_translucency</font>**(<font color="#00a6ed">**self**</font>, <font color="#00a6ed">**enable**</font>)  
+Sets CARLA's behavior whether the semantic + instance segmentation and depth annotations traverse translucent materials or not. To find the closest obstacle, this should be set to false (default). If you are interested in annotations that detect objects behind windows, this should be enabled.  
+    - **Parameters:**
+        - `enable` (_bool_) - Enables or disables the traversal of translucent materials in semantic, instance and depth annotations. __Default is `False`__.  
 - <a name="carla.World.set_pedestrians_cross_factor"></a>**<font color="#7fb800">set_pedestrians_cross_factor</font>**(<font color="#00a6ed">**self**</font>, <font color="#00a6ed">**percentage**</font>)  
     - **Parameters:**
         - `percentage` (_float_) - Sets the percentage of pedestrians that can walk on the road or cross at any point on the road. Value should be between `0.0` and `1.0`. For example, a value of `0.1` would allow 10% of pedestrians to walk on the road. __Default is `0.0`__.  

LibCarla/source/carla/client/World.cpp

@@ -394,5 +394,10 @@ namespace client {
     }
   }
 
+  void World::SetAnnotationsTraverseTranslucency(
+      bool enable) {
+    _episode.Lock()->SetAnnotationsTraverseTranslucency(enable);
+  }
+
 } // namespace client
 } // namespace carla

LibCarla/source/carla/client/World.h

@@ -96,7 +96,7 @@ namespace client {
 
     /// Get Gravity value used for IMUI Sensor accelerometer calculation
     float GetIMUISensorGravity() const;
-    
+
     /// Set Gravity value used for IMUI Sensor accelerometer calculation
     void SetIMUISensorGravity(float NewIMUISensorGravity);
 
@@ -237,6 +237,10 @@ namespace client {
         const rpc::TextureFloatColor& normal_texture,
         const rpc::TextureFloatColor& ao_roughness_metallic_emissive_texture);
 
+    /// Enables or disables the traversal of translucent materials in semantic, instance and depth annotations. Defaults to False.
+    void SetAnnotationsTraverseTranslucency(
+        bool enable);
+
     std::vector<std::string> GetNamesOfAllObjects() const;
 
   private:

LibCarla/source/carla/client/detail/Client.cpp

@@ -773,6 +773,11 @@ namespace detail {
     return _pimpl->CallAndWait<return_t>("cast_ray", start_location, end_location);
   }
 
+  void Client::SetAnnotationsTraverseTranslucency(
+      bool enable) {
+    _pimpl->AsyncCall("set_annotations_traverse_translucency", enable);
+  }
+
 } // namespace detail
 } // namespace client
 } // namespace carla

LibCarla/source/carla/client/detail/Client.h

@@ -478,6 +478,8 @@ namespace detail {
     std::vector<rpc::LabelledPoint> CastRay(
         geom::Location start_location, geom::Location end_location) const;
 
+    void SetAnnotationsTraverseTranslucency(
+        bool enable);
   private:
 
     class Pimpl;

LibCarla/source/carla/client/detail/Simulator.h

@@ -300,6 +300,11 @@ namespace detail {
       return _client.CastRay(start_location, end_location);
     }
 
+    void SetAnnotationsTraverseTranslucency(
+      bool enable) {
+        _client.SetAnnotationsTraverseTranslucency(enable);
+      }
+
     /// @}
     // =========================================================================
     /// @name AI

PythonAPI/carla/source/carla/libcarla.pyi

@@ -8056,6 +8056,15 @@ class World:
     # endregion
 
     # region Setters
+    def set_annotations_traverse_translucency(self, enable: bool):
+        """Sets CARLA's behavior whether the semantic + instance segmentation and depth annotations traverse translucent materials or not.
+        To find the closest obstacle, this should be set to false (default). If you are interested in annotations that detect objects
+        behind windows, this should be enabled.
+
+        Args:
+            `enable (bool)`: Enables or disables the traversal of translucent materials in semantic, instance and depth annotations. Defaults to False.
+        """
+
     def set_pedestrians_cross_factor(self, percentage: float):
         """Sets the percentage of pedestrians that can walk on the road or cross at any point on the road.
 

PythonAPI/carla/source/libcarla/World.cpp

@@ -363,6 +363,7 @@ void export_world() {
         self.ApplyTexturesToObjects(PythonLitstToVector<std::string>(list), diffuse_texture, emissive_texture, normal_texture, ao_roughness_metallic_emissive_texture);
       }, (arg("objects_name_list"), arg("diffuse_texture"), arg("emissive_texture"), arg("normal_texture"), arg("ao_roughness_metallic_emissive_texture")))
     .def(self_ns::str(self_ns::self))
+    .def("set_annotations_traverse_translucency", CALL_WITHOUT_GIL_1(cc::World, SetAnnotationsTraverseTranslucency, bool), (arg("enable")))
   ;
 
 #undef SPAWN_ACTOR_WITHOUT_GIL

PythonAPI/docs/world.yml

@@ -814,6 +814,15 @@
         If the layer is already unloaded the call has no effect.
       warning: This only affects "Opt" maps. The minimum layout includes roads, sidewalks, traffic lights and traffic signs.
     # --------------------------------------
+    - def_name: set_annotations_traverse_translucency
+      params:
+      - param_name: enable
+        type: bool
+        doc: >
+          Enables or disables the traversal of translucent materials in semantic, instance and depth annotations. __Default is `False`__.
+      doc: >
+        Sets CARLA's behavior whether the semantic + instance segmentation and depth annotations traverse translucent materials or not. To find the closest obstacle, this should be set to false (default). If you are interested in annotations that detect objects behind windows, this should be enabled.
+    # --------------------------------------
     - def_name: set_pedestrians_cross_factor
       params:
       - param_name: percentage

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/TaggedMaterials.cpp

@@ -15,6 +15,10 @@ UTaggedMaterialsRegistry::UTaggedMaterialsRegistry() {
   static ConstructorHelpers::FObjectFinder<UMaterial> TaggedOpaqueMaterialObject(*MaterialPath);
   // TODO: Replace with VertexColorViewModeMaterial_ColorOnly?
   TaggedOpaqueMaterial = TaggedOpaqueMaterialObject.Object;
+
+  FString TaggingMPCPath = TEXT("Material'/Carla/PostProcessingMaterials/MPC/MPC_Tagging.MPC_Tagging'");
+  static ConstructorHelpers::FObjectFinder<UMaterialParameterCollection> TaggingMPCObject(*TaggingMPCPath);
+  TaggingParamerCollection = TaggingMPCObject.Object;
 }
 
 UTaggedMaterialsRegistry* UTaggedMaterialsRegistry::Create(const FString& RegistryName) {
@@ -60,11 +64,24 @@ UTaggedMaterialsRegistry* UTaggedMaterialsRegistry::Get() {
   return Registry;
 }
 
+void UTaggedMaterialsRegistry::SetTaggingTraverseTranslucency(UCarlaEpisode* Episode, bool bTaggingTraverseTranslucency) {
+  UMaterialParameterCollectionInstance* MPCInstance = Episode->GetWorld()->GetParameterCollectionInstance(TaggingParamerCollection);
+  MPCInstance->SetScalarParameterValue("TraverseTranslucency", bTaggingTraverseTranslucency);
+}
+
 UMaterialInstanceDynamic* UTaggedMaterialsRegistry::GetTaggedMaterial() {
   return UMaterialInstanceDynamic::Create(TaggedOpaqueMaterial, this);
 }
 
 UMaterialInstanceDynamic* UTaggedMaterialsRegistry::GetTaggedMaterial(UMaterialInterface* UsedMaterial) {
+  // If UsedMaterial is a translucent material, we use the default tagged material but enable IsTranslucent.
+  // Using SetTaggingTraverseTranslucency, it can be globally enabled/disabled that these materials appear invisible or not.
+  if (UsedMaterial && UsedMaterial->GetBlendMode() == EBlendMode::BLEND_Translucent) {
+    UMaterialInstanceDynamic* TranslucentMaterial = GetTaggedMaterial();
+    TranslucentMaterial->SetScalarParameterValue("IsTranslucent", true);
+    return TranslucentMaterial;
+  }
+
   // If UsedMaterial is null OR neither masked nor using WorldPositionOffset, we return NULL to indicate,
   // that the requested material does not require a fine-grained tag-injected correspondence.
   if (!UsedMaterial || !(UsedMaterial->IsMasked() || UsedMaterial->GetMaterial()->WorldPositionOffset.IsConnected())) {

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Game/TaggedMaterials.h

@@ -17,6 +17,7 @@ class CARLA_API UTaggedMaterialsRegistry : public UObject
 
   UMaterialInstanceDynamic* GetTaggedMaterial();
   UMaterialInstanceDynamic* GetTaggedMaterial(UMaterialInterface* UsedMaterial);
+  void SetTaggingTraverseTranslucency(UCarlaEpisode* Episode, bool bTaggingTraverseTranslucency);
   FORCEINLINE int32 Num() const {return TaggedMaskedMaterials.Num();}
 
 #if WITH_EDITOR
@@ -48,6 +49,9 @@ class CARLA_API UTaggedMaterialsRegistry : public UObject
 
   UPROPERTY(VisibleAnywhere)
   TMap<FString, UMaterialInterface*> TaggedMaskedMaterials;
+
+  UPROPERTY(VisibleAnywhere)
+  UMaterialParameterCollection* TaggingParamerCollection;
 
   bool bPendingChanges = false;
 

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/DepthCamera.cpp

@@ -6,7 +6,7 @@
 
 #include "Carla.h"
 #include "Carla/Sensor/DepthCamera.h"
-
+#include "Carla/Sensor/InstanceSegmentationCamera.h"
 #include "Carla/Actor/ActorBlueprintFunctionLibrary.h"
 
 #include "Carla/Sensor/PixelReader.h"
@@ -30,8 +30,16 @@ ADepthCamera::ADepthCamera(const FObjectInitializer &ObjectInitializer)
   );
 }
 
+void ADepthCamera::SetUpSceneCaptureComponent(USceneCaptureComponent2D &SceneCapture)
+{
+  Super::SetUpSceneCaptureComponent(SceneCapture);
+  AInstanceSegmentationCamera::SetUpSceneCaptureComponentForTagging(SceneCapture);
+}
+
 void ADepthCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds)
 {
   TRACE_CPUPROFILER_EVENT_SCOPE(ADepthCamera::PostPhysTick);
+
+  AInstanceSegmentationCamera::PostPhysTickForTagging(*GetCaptureComponent2D());
   FPixelReader::SendPixelsInRenderThread<ADepthCamera, FColor>(*this);
-}
+}

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/DepthCamera.h

@@ -26,5 +26,6 @@ class CARLA_API ADepthCamera : public AShaderBasedSensor
 
 protected:
 
+  void SetUpSceneCaptureComponent(USceneCaptureComponent2D &SceneCapture) override;
   void PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds) override;
 };

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/InstanceSegmentationCamera.cpp

@@ -30,41 +30,37 @@ AInstanceSegmentationCamera::AInstanceSegmentationCamera(
 void AInstanceSegmentationCamera::SetUpSceneCaptureComponent(USceneCaptureComponent2D &SceneCapture)
 {
   Super::SetUpSceneCaptureComponent(SceneCapture);
-
   ApplyViewMode(VMI_Unlit, true, SceneCapture.ShowFlags);
+  SetUpSceneCaptureComponentForTagging(SceneCapture);
+}
 
-  SceneCapture.ShowFlags.SetNotDrawTaggedComponents(false); // TaggedComponent detects this and sets view relevance for proxy material
+void AInstanceSegmentationCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds)
+{
+  TRACE_CPUPROFILER_EVENT_SCOPE(AInstanceSegmentationCamera::PostPhysTick);
 
-  SceneCapture.ShowFlags.SetAtmosphere(false);
+  PostPhysTickForTagging(*GetCaptureComponent2D());
+  FPixelReader::SendPixelsInRenderThread<AInstanceSegmentationCamera, FColor>(*this);
+}
 
+void AInstanceSegmentationCamera::SetUpSceneCaptureComponentForTagging(USceneCaptureComponent2D &SceneCapture) {
+  SceneCapture.ShowFlags.SetNotDrawTaggedComponents(false); // TaggedComponent detects this and sets view relevance for proxy material
+  SceneCapture.ShowFlags.SetAtmosphere(false);
   SceneCapture.PrimitiveRenderMode = ESceneCapturePrimitiveRenderMode::PRM_UseShowOnlyList;
-
-  TArray<UObject *> TaggedComponents;
-  GetObjectsOfClass(UTaggedComponent::StaticClass(), TaggedComponents, false, EObjectFlags::RF_ClassDefaultObject, EInternalObjectFlags::AllFlags);
-  GetObjectsOfClass(UTaggedLandscapeComponent::StaticClass(), TaggedComponents, false, EObjectFlags::RF_ClassDefaultObject, EInternalObjectFlags::AllFlags);
-
-  TArray<UPrimitiveComponent *> ShowOnlyComponents;
-  for (UObject *Object : TaggedComponents) {
-    UPrimitiveComponent *Component = Cast<UPrimitiveComponent>(Object);
-    SceneCapture.ShowOnlyComponents.Emplace(Component);
-  }
+  AddTaggedComponentsToSceneCapture(SceneCapture);
 }
 
-void AInstanceSegmentationCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds)
-{
-  TRACE_CPUPROFILER_EVENT_SCOPE(AInstanceSegmentationCamera::PostPhysTick);
+void AInstanceSegmentationCamera::PostPhysTickForTagging(USceneCaptureComponent2D &SceneCapture) {
+  SceneCapture.ClearShowOnlyComponents();
+  AddTaggedComponentsToSceneCapture(SceneCapture);
+}
 
-  USceneCaptureComponent2D* SceneCapture = GetCaptureComponent2D();
-  TArray<UObject *> TaggedComponents;
+void AInstanceSegmentationCamera::AddTaggedComponentsToSceneCapture(USceneCaptureComponent2D &SceneCapture) {
+  TArray<UObject*> TaggedComponents;
   GetObjectsOfClass(UTaggedComponent::StaticClass(), TaggedComponents, false, EObjectFlags::RF_ClassDefaultObject, EInternalObjectFlags::AllFlags);
   GetObjectsOfClass(UTaggedLandscapeComponent::StaticClass(), TaggedComponents, false, EObjectFlags::RF_ClassDefaultObject, EInternalObjectFlags::AllFlags);
 
-  SceneCapture->ClearShowOnlyComponents();
-  for (UObject *Object : TaggedComponents) {
-    UPrimitiveComponent *Component = Cast<UPrimitiveComponent>(Object);
-    SceneCapture->ShowOnlyComponents.Emplace(Component);
+  for (UObject* Object : TaggedComponents) {
+    UPrimitiveComponent* Component = Cast<UPrimitiveComponent>(Object);
+    SceneCapture.ShowOnlyComponents.Emplace(Component);
   }
-
-  FPixelReader::SendPixelsInRenderThread<AInstanceSegmentationCamera, FColor>(*this);
-
 }

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/InstanceSegmentationCamera.h

@@ -24,8 +24,12 @@ class CARLA_API AInstanceSegmentationCamera : public AShaderBasedSensor
 
   AInstanceSegmentationCamera(const FObjectInitializer &ObjectInitializer);
 
+  static void SetUpSceneCaptureComponentForTagging(USceneCaptureComponent2D &SceneCapture);
+  static void PostPhysTickForTagging(USceneCaptureComponent2D &SceneCapture);
+
 protected:
 
   void SetUpSceneCaptureComponent(USceneCaptureComponent2D &SceneCapture) override;
   void PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds) override;
+  static void AddTaggedComponentsToSceneCapture(USceneCaptureComponent2D &SceneCapture);
 };

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SemanticSegmentationCamera.cpp

@@ -6,6 +6,7 @@
 
 #include "Carla.h"
 #include "Carla/Sensor/SemanticSegmentationCamera.h"
+#include "Carla/Sensor/InstanceSegmentationCamera.h"
 
 #include "Actor/ActorBlueprintFunctionLibrary.h"
 
@@ -26,8 +27,16 @@ ASemanticSegmentationCamera::ASemanticSegmentationCamera(
       TEXT("Material'/Carla/PostProcessingMaterials/GTMaterial.GTMaterial'"));
 }
 
+void ASemanticSegmentationCamera::SetUpSceneCaptureComponent(USceneCaptureComponent2D &SceneCapture)
+{
+  Super::SetUpSceneCaptureComponent(SceneCapture);
+  AInstanceSegmentationCamera::SetUpSceneCaptureComponentForTagging(SceneCapture);
+}
+
 void ASemanticSegmentationCamera::PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds)
 {
   TRACE_CPUPROFILER_EVENT_SCOPE(ASemanticSegmentationCamera::PostPhysTick);
+
+  AInstanceSegmentationCamera::PostPhysTickForTagging(*GetCaptureComponent2D());
   FPixelReader::SendPixelsInRenderThread<ASemanticSegmentationCamera, FColor>(*this);
 }

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Sensor/SemanticSegmentationCamera.h

@@ -26,5 +26,6 @@ class CARLA_API ASemanticSegmentationCamera : public AShaderBasedSensor
 
 protected:
 
+  void SetUpSceneCaptureComponent(USceneCaptureComponent2D &SceneCapture) override;
   void PostPhysTick(UWorld *World, ELevelTick TickType, float DeltaSeconds) override;
 };

Unreal/CarlaUE4/Plugins/Carla/Source/Carla/Server/CarlaServer.cpp

@@ -26,6 +26,7 @@
 #include "Carla/Walker/WalkerBase.h"
 #include "GameFramework/CharacterMovementComponent.h"
 #include "Carla/Game/Tagger.h"
+#include "Carla/Game/TaggedMaterials.h"
 #include "Carla/Game/CarlaStatics.h"
 #include "Carla/Vehicle/MovementComponents/CarSimManagerComponent.h"
 #include "Carla/Vehicle/MovementComponents/ChronoMovementComponent.h"
@@ -643,6 +644,13 @@ void FCarlaServer::FPimpl::BindActions()
     return R<void>::Success();
   };
 
+  BIND_SYNC(set_annotations_traverse_translucency) << [this](bool Enable) -> R<void>
+  {
+    REQUIRE_CARLA_EPISODE();
+    UTaggedMaterialsRegistry::Get()->SetTaggingTraverseTranslucency(Episode, Enable);
+    return R<void>::Success();
+  };
+
   // ~~ Weather ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
   BIND_SYNC(get_weather_parameters) << [this]() -> R<cr::WeatherParameters>