diff --git a/webrtc-sys/build.rs b/webrtc-sys/build.rs index d4c2e70f9..87213e82a 100644 --- a/webrtc-sys/build.rs +++ b/webrtc-sys/build.rs @@ -191,7 +191,9 @@ fn main() { .file("src/nvidia/NvCodec/NvCodec/NvEncoder/NvEncoder.cpp") .file("src/nvidia/NvCodec/NvCodec/NvEncoder/NvEncoderCuda.cpp") .file("src/nvidia/h264_encoder_impl.cpp") + .file("src/nvidia/h265_encoder_impl.cpp") .file("src/nvidia/h264_decoder_impl.cpp") + .file("src/nvidia/h265_decoder_impl.cpp") .file("src/nvidia/nvidia_decoder_factory.cpp") .file("src/nvidia/nvidia_encoder_factory.cpp") .file("src/nvidia/cuda_context.cpp") diff --git a/webrtc-sys/src/nvidia/h264_encoder_impl.cpp b/webrtc-sys/src/nvidia/h264_encoder_impl.cpp index dade3f0d3..f6cc62b24 100644 --- a/webrtc-sys/src/nvidia/h264_encoder_impl.cpp +++ b/webrtc-sys/src/nvidia/h264_encoder_impl.cpp @@ -238,6 +238,11 @@ int32_t NvidiaH264EncoderImpl::InitEncode( return WEBRTC_VIDEO_CODEC_ERROR; } + RTC_LOG(LS_INFO) << "NVIDIA H264 NVENC initialized: " + << codec_.width << "x" << codec_.height + << " @ " << codec_.maxFramerate << "fps, target_bps=" + << configuration_.target_bps; + SimulcastRateAllocator init_allocator(env_, codec_); VideoBitrateAllocation allocation = init_allocator.Allocate(VideoBitrateAllocationParameters( diff --git a/webrtc-sys/src/nvidia/h265_decoder_impl.cpp b/webrtc-sys/src/nvidia/h265_decoder_impl.cpp new file mode 100644 index 000000000..d2fa23831 --- /dev/null +++ b/webrtc-sys/src/nvidia/h265_decoder_impl.cpp @@ -0,0 +1,162 @@ +#include "h265_decoder_impl.h" + +#include +#include +#include +#include + +#include "NvDecoder/NvDecoder.h" +#include "Utils/NvCodecUtils.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" + +namespace webrtc { + +static ColorSpace ExtractColorSpaceFromFormat(const CUVIDEOFORMAT& format) { + return ColorSpace( + static_cast( + format.video_signal_description.color_primaries), + static_cast( + format.video_signal_description.transfer_characteristics), + static_cast( + format.video_signal_description.matrix_coefficients), + static_cast( + format.video_signal_description.video_full_range_flag)); +} + +NvidiaH265DecoderImpl::NvidiaH265DecoderImpl(CUcontext context) + : cu_context_(context), + decoder_(nullptr), + is_configured_decoder_(false), + decoded_complete_callback_(nullptr), + buffer_pool_(false) {} + +NvidiaH265DecoderImpl::~NvidiaH265DecoderImpl() { Release(); } + +VideoDecoder::DecoderInfo NvidiaH265DecoderImpl::GetDecoderInfo() const { + VideoDecoder::DecoderInfo info; + info.implementation_name = "NVIDIA H265 Decoder"; + info.is_hardware_accelerated = true; + return info; +} + +bool NvidiaH265DecoderImpl::Configure(const Settings& settings) { + if (settings.codec_type() != kVideoCodecH265) { + RTC_LOG(LS_ERROR) << "initialization failed: codec type is not H265"; + return false; + } + if (!settings.max_render_resolution().Valid()) { + RTC_LOG(LS_ERROR) + << "initialization failed on codec_settings width < 0 or height < 0"; + return false; + } + + settings_ = settings; + + const CUresult result = cuCtxSetCurrent(cu_context_); + if (!ck(result)) { + RTC_LOG(LS_ERROR) << "initialization failed on cuCtxSetCurrent result" + << result; + return false; + } + + int maxWidth = 4096; + int maxHeight = 4096; + + decoder_ = std::make_unique( + cu_context_, false, cudaVideoCodec_HEVC, true, false, nullptr, nullptr, + false, maxWidth, maxHeight); + return true; +} + +int32_t NvidiaH265DecoderImpl::RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) { + decoded_complete_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t NvidiaH265DecoderImpl::Release() { + buffer_pool_.Release(); + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t NvidiaH265DecoderImpl::Decode(const EncodedImage& input_image, + bool /*missing_frames*/, + int64_t /*render_time_ms*/) { + CUcontext current; + if (!ck(cuCtxGetCurrent(¤t))) { + RTC_LOG(LS_ERROR) << "decode failed on cuCtxGetCurrent"; + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (current != cu_context_) { + RTC_LOG(LS_ERROR) + << "decode failed: current context does not match held context"; + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (decoded_complete_callback_ == nullptr) { + RTC_LOG(LS_ERROR) << "decode failed: decoded_complete_callback_ not set"; + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (!input_image.data() || !input_image.size()) { + RTC_LOG(LS_ERROR) << "decode failed: input image is null"; + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + int nFrameReturned = 0; + do { + nFrameReturned = decoder_->Decode( + input_image.data(), static_cast(input_image.size()), + CUVID_PKT_TIMESTAMP, input_image.RtpTimestamp()); + } while (nFrameReturned == 0); + + is_configured_decoder_ = true; + + if (decoder_->GetOutputFormat() != cudaVideoSurfaceFormat_NV12) { + RTC_LOG(LS_ERROR) << "not supported output format: " + << decoder_->GetOutputFormat(); + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + const ColorSpace& color_space = + input_image.ColorSpace() ? *input_image.ColorSpace() + : ExtractColorSpaceFromFormat( + decoder_->GetVideoFormatInfo()); + + for (int i = 0; i < nFrameReturned; i++) { + int64_t timeStamp; + uint8_t* pFrame = decoder_->GetFrame(&timeStamp); + + webrtc::scoped_refptr i420_buffer = + buffer_pool_.CreateI420Buffer(decoder_->GetWidth(), + decoder_->GetHeight()); + + int result = libyuv::NV12ToI420( + pFrame, decoder_->GetDeviceFramePitch(), + pFrame + decoder_->GetHeight() * decoder_->GetDeviceFramePitch(), + decoder_->GetDeviceFramePitch(), i420_buffer->MutableDataY(), + i420_buffer->StrideY(), i420_buffer->MutableDataU(), + i420_buffer->StrideU(), i420_buffer->MutableDataV(), + i420_buffer->StrideV(), decoder_->GetWidth(), decoder_->GetHeight()); + + if (result) { + RTC_LOG(LS_INFO) << "libyuv::NV12ToI420 failed. error:" << result; + } + + VideoFrame decoded_frame = VideoFrame::Builder() + .set_video_frame_buffer(i420_buffer) + .set_timestamp_rtp(static_cast( + timeStamp)) + .set_color_space(color_space) + .build(); + + std::optional decodetime; + std::optional qp; // Not parsed for H265 currently + decoded_complete_callback_->Decoded(decoded_frame, decodetime, qp); + } + + return WEBRTC_VIDEO_CODEC_OK; +} + +} // end namespace webrtc + + diff --git a/webrtc-sys/src/nvidia/h265_decoder_impl.h b/webrtc-sys/src/nvidia/h265_decoder_impl.h new file mode 100644 index 000000000..54dcbfc20 --- /dev/null +++ b/webrtc-sys/src/nvidia/h265_decoder_impl.h @@ -0,0 +1,47 @@ +#ifndef WEBRTC_NVIDIA_H265_DECODER_IMPL_H_ +#define WEBRTC_NVIDIA_H265_DECODER_IMPL_H_ + +#include +#include +#include +#include +#include +#include +#include + +#include "NvDecoder/NvDecoder.h" + +namespace webrtc { + +class NvidiaH265DecoderImpl : public VideoDecoder { + public: + explicit NvidiaH265DecoderImpl(CUcontext context); + NvidiaH265DecoderImpl(const NvidiaH265DecoderImpl&) = delete; + NvidiaH265DecoderImpl& operator=(const NvidiaH265DecoderImpl&) = delete; + ~NvidiaH265DecoderImpl() override; + + bool Configure(const Settings& settings) override; + int32_t Decode(const EncodedImage& input_image, + bool missing_frames, + int64_t render_time_ms) override; + int32_t RegisterDecodeCompleteCallback( + DecodedImageCallback* callback) override; + int32_t Release() override; + DecoderInfo GetDecoderInfo() const override; + + private: + CUcontext cu_context_; + std::unique_ptr decoder_; + bool is_configured_decoder_; + + Settings settings_; + + DecodedImageCallback* decoded_complete_callback_ = nullptr; + webrtc::VideoFrameBufferPool buffer_pool_; +}; + +} // end namespace webrtc + +#endif // WEBRTC_NVIDIA_H265_DECODER_IMPL_H_ + + diff --git a/webrtc-sys/src/nvidia/h265_encoder_impl.cpp b/webrtc-sys/src/nvidia/h265_encoder_impl.cpp new file mode 100644 index 000000000..8a33076d3 --- /dev/null +++ b/webrtc-sys/src/nvidia/h265_encoder_impl.cpp @@ -0,0 +1,381 @@ +#include "h265_encoder_impl.h" + +#include +#include +#include + +#include "absl/strings/match.h" +#include "absl/types/optional.h" +#include "api/video/video_codec_constants.h" +#include "api/video_codecs/scalability_mode.h" +#include "common_video/libyuv/include/webrtc_libyuv.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "modules/video_coding/svc/create_scalability_structure.h" +#include "modules/video_coding/utility/simulcast_rate_allocator.h" +#include "modules/video_coding/utility/simulcast_utility.h" +#include "rtc_base/checks.h" +#include "rtc_base/logging.h" +#include "rtc_base/time_utils.h" +#include "system_wrappers/include/metrics.h" +#include "third_party/libyuv/include/libyuv/convert.h" +#include "third_party/libyuv/include/libyuv/scale.h" + +namespace webrtc { + +// Used by histograms. Values of entries should not be changed. +enum H265EncoderImplEvent { + kH265EncoderEventInit = 0, + kH265EncoderEventError = 1, + kH265EncoderEventMax = 16, +}; + +NvidiaH265EncoderImpl::NvidiaH265EncoderImpl( + const webrtc::Environment& env, + CUcontext context, + CUmemorytype memory_type, + NV_ENC_BUFFER_FORMAT nv_format, + const SdpVideoFormat& format) + : env_(env), + encoder_(nullptr), + cu_context_(context), + cu_memory_type_(memory_type), + cu_scaled_array_(nullptr), + nv_format_(nv_format), + format_(format) { + RTC_CHECK_NE(cu_memory_type_, CU_MEMORYTYPE_HOST); +} + +NvidiaH265EncoderImpl::~NvidiaH265EncoderImpl() { + Release(); +} + +void NvidiaH265EncoderImpl::ReportInit() { + if (has_reported_init_) + return; + RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H265EncoderImpl.Event", + kH265EncoderEventInit, kH265EncoderEventMax); + has_reported_init_ = true; +} + +void NvidiaH265EncoderImpl::ReportError() { + if (has_reported_error_) + return; + RTC_HISTOGRAM_ENUMERATION("WebRTC.Video.H265EncoderImpl.Event", + kH265EncoderEventError, kH265EncoderEventMax); + has_reported_error_ = true; +} + +int32_t NvidiaH265EncoderImpl::InitEncode( + const VideoCodec* inst, + const VideoEncoder::Settings& settings) { + if (!inst || inst->codecType != kVideoCodecH265) { + ReportError(); + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (inst->maxFramerate == 0) { + ReportError(); + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (inst->width < 1 || inst->height < 1) { + ReportError(); + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + + int32_t release_ret = Release(); + if (release_ret != WEBRTC_VIDEO_CODEC_OK) { + ReportError(); + return release_ret; + } + + codec_ = *inst; + + if (codec_.numberOfSimulcastStreams == 0) { + codec_.simulcastStream[0].width = codec_.width; + codec_.simulcastStream[0].height = codec_.height; + } + + const size_t new_capacity = + CalcBufferSize(VideoType::kI420, codec_.width, codec_.height); + encoded_image_.SetEncodedData(EncodedImageBuffer::Create(new_capacity)); + encoded_image_._encodedWidth = codec_.width; + encoded_image_._encodedHeight = codec_.height; + encoded_image_.set_size(0); + + configuration_.sending = false; + configuration_.frame_dropping_on = codec_.GetFrameDropEnabled(); + configuration_.key_frame_interval = 0; + + configuration_.width = codec_.width; + configuration_.height = codec_.height; + + configuration_.max_frame_rate = codec_.maxFramerate; + configuration_.target_bps = codec_.startBitrate * 1000; + configuration_.max_bps = codec_.maxBitrate * 1000; + + const CUresult result = cuCtxSetCurrent(cu_context_); + if (result != CUDA_SUCCESS) { + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + + try { + if (cu_memory_type_ == CU_MEMORYTYPE_DEVICE) { + encoder_ = std::make_unique(cu_context_, codec_.width, + codec_.height, nv_format_, 0); + } else { + RTC_DCHECK_NOTREACHED(); + } + } catch (const NVENCException& e) { + RTC_LOG(LS_ERROR) << "Failed Initialize NvEncoder " << e.what(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + nv_initialize_params_.version = NV_ENC_INITIALIZE_PARAMS_VER; + nv_encode_config_.version = NV_ENC_CONFIG_VER; + nv_initialize_params_.encodeConfig = &nv_encode_config_; + + GUID encodeGuid = NV_ENC_CODEC_HEVC_GUID; + GUID presetGuid = NV_ENC_PRESET_P4_GUID; + + encoder_->CreateDefaultEncoderParams(&nv_initialize_params_, encodeGuid, + presetGuid, + NV_ENC_TUNING_INFO_ULTRA_LOW_LATENCY); + + nv_initialize_params_.frameRateNum = + static_cast(configuration_.max_frame_rate); + nv_initialize_params_.frameRateDen = 1; + nv_initialize_params_.bufferFormat = nv_format_; + + nv_encode_config_.profileGUID = NV_ENC_HEVC_PROFILE_MAIN_GUID; + nv_encode_config_.gopLength = NVENC_INFINITE_GOPLENGTH; + nv_encode_config_.frameIntervalP = 1; + nv_encode_config_.rcParams.version = NV_ENC_RC_PARAMS_VER; + nv_encode_config_.rcParams.rateControlMode = NV_ENC_PARAMS_RC_CBR; + nv_encode_config_.rcParams.averageBitRate = configuration_.target_bps; + nv_encode_config_.rcParams.vbvBufferSize = + (nv_encode_config_.rcParams.averageBitRate * + nv_initialize_params_.frameRateDen / + nv_initialize_params_.frameRateNum) * + 5; + nv_encode_config_.rcParams.vbvInitialDelay = + nv_encode_config_.rcParams.vbvBufferSize; + + try { + encoder_->CreateEncoder(&nv_initialize_params_); + } catch (const NVENCException& e) { + RTC_LOG(LS_ERROR) << "Failed Initialize NvEncoder " << e.what(); + return WEBRTC_VIDEO_CODEC_ERROR; + } + + RTC_LOG(LS_INFO) << "NVIDIA H265/HEVC NVENC initialized: " + << codec_.width << "x" << codec_.height + << " @ " << codec_.maxFramerate << "fps, target_bps=" + << configuration_.target_bps; + + SimulcastRateAllocator init_allocator(env_, codec_); + VideoBitrateAllocation allocation = + init_allocator.Allocate(VideoBitrateAllocationParameters( + DataRate::KilobitsPerSec(codec_.startBitrate), codec_.maxFramerate)); + SetRates(RateControlParameters(allocation, codec_.maxFramerate)); + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t NvidiaH265EncoderImpl::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + encoded_image_callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t NvidiaH265EncoderImpl::Release() { + if (encoder_) { + encoder_->DestroyEncoder(); + encoder_ = nullptr; + } + if (cu_scaled_array_) { + cuArrayDestroy(cu_scaled_array_); + cu_scaled_array_ = nullptr; + } + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t NvidiaH265EncoderImpl::Encode( + const VideoFrame& input_frame, + const std::vector* frame_types) { + if (!encoder_) { + ReportError(); + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + if (!encoded_image_callback_) { + RTC_LOG(LS_WARNING) + << "InitEncode() has been called, but a callback function " + "has not been set with RegisterEncodeCompleteCallback()"; + ReportError(); + return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + } + + webrtc::scoped_refptr frame_buffer = + input_frame.video_frame_buffer()->ToI420(); + if (!frame_buffer) { + RTC_LOG(LS_ERROR) << "Failed to convert " + << VideoFrameBufferTypeToString( + input_frame.video_frame_buffer()->type()) + << " image to I420. Can't encode frame."; + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + RTC_CHECK(frame_buffer->type() == VideoFrameBuffer::Type::kI420); + + bool is_keyframe_needed = false; + if (configuration_.key_frame_request && configuration_.sending) { + is_keyframe_needed = true; + } + + bool send_key_frame = + is_keyframe_needed || + (frame_types && (*frame_types)[0] == VideoFrameType::kVideoFrameKey); + if (send_key_frame) { + is_keyframe_needed = true; + configuration_.key_frame_request = false; + } + + RTC_DCHECK_EQ(configuration_.width, frame_buffer->width()); + RTC_DCHECK_EQ(configuration_.height, frame_buffer->height()); + + if (!configuration_.sending) { + return WEBRTC_VIDEO_CODEC_NO_OUTPUT; + } + + if (frame_types != nullptr) { + if ((*frame_types)[0] == VideoFrameType::kEmptyFrame) { + return WEBRTC_VIDEO_CODEC_NO_OUTPUT; + } + } + + try { + const NvEncInputFrame* nv_enc_input_frame = encoder_->GetNextInputFrame(); + + if (cu_memory_type_ == CU_MEMORYTYPE_DEVICE) { + NvEncoderCuda::CopyToDeviceFrame( + cu_context_, (void*)frame_buffer->DataY(), frame_buffer->StrideY(), + reinterpret_cast(nv_enc_input_frame->inputPtr), + nv_enc_input_frame->pitch, input_frame.width(), input_frame.height(), + CU_MEMORYTYPE_HOST, nv_enc_input_frame->bufferFormat, + nv_enc_input_frame->chromaOffsets, nv_enc_input_frame->numChromaPlanes); + } + + NV_ENC_PIC_PARAMS pic_params = NV_ENC_PIC_PARAMS(); + pic_params.version = NV_ENC_PIC_PARAMS_VER; + pic_params.encodePicFlags = 0; + if (is_keyframe_needed) { + pic_params.encodePicFlags = NV_ENC_PIC_FLAG_FORCEINTRA | + NV_ENC_PIC_FLAG_FORCEIDR | + NV_ENC_PIC_FLAG_OUTPUT_SPSPPS; + configuration_.key_frame_request = false; + } + + current_encoding_is_keyframe_ = is_keyframe_needed; + + std::vector> bit_stream; + encoder_->EncodeFrame(bit_stream, &pic_params); + + for (std::vector& packet : bit_stream) { + int32_t result = ProcessEncodedFrame(packet, input_frame); + if (result != WEBRTC_VIDEO_CODEC_OK) { + return result; + } + } + current_encoding_is_keyframe_ = false; + } catch (const NVENCException& e) { + RTC_LOG(LS_ERROR) << "Failed EncodeFrame NvEncoder " << e.what(); + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + + return WEBRTC_VIDEO_CODEC_OK; +} + +int32_t NvidiaH265EncoderImpl::ProcessEncodedFrame( + std::vector& packet, + const ::webrtc::VideoFrame& inputFrame) { + encoded_image_._encodedWidth = encoder_->GetEncodeWidth(); + encoded_image_._encodedHeight = encoder_->GetEncodeHeight(); + encoded_image_.SetRtpTimestamp(inputFrame.rtp_timestamp()); + encoded_image_.SetSimulcastIndex(0); + encoded_image_.ntp_time_ms_ = inputFrame.ntp_time_ms(); + encoded_image_.capture_time_ms_ = inputFrame.render_time_ms(); + encoded_image_.rotation_ = inputFrame.rotation(); + encoded_image_.content_type_ = VideoContentType::UNSPECIFIED; + encoded_image_.timing_.flags = VideoSendTiming::kInvalid; + encoded_image_._frameType = + current_encoding_is_keyframe_ ? VideoFrameType::kVideoFrameKey + : VideoFrameType::kVideoFrameDelta; + encoded_image_.SetColorSpace(inputFrame.color_space()); + + encoded_image_.SetEncodedData( + EncodedImageBuffer::Create(packet.data(), packet.size())); + encoded_image_.set_size(packet.size()); + + encoded_image_.qp_ = -1; + + CodecSpecificInfo codecInfo; + codecInfo.codecType = kVideoCodecH265; + + const auto result = + encoded_image_callback_->OnEncodedImage(encoded_image_, &codecInfo); + if (result.error != EncodedImageCallback::Result::OK) { + RTC_LOG(LS_ERROR) << "Encode m_encodedCompleteCallback failed " + << result.error; + return WEBRTC_VIDEO_CODEC_ERROR; + } + return WEBRTC_VIDEO_CODEC_OK; +} + +VideoEncoder::EncoderInfo NvidiaH265EncoderImpl::GetEncoderInfo() const { + EncoderInfo info; + info.supports_native_handle = false; + info.implementation_name = "NVIDIA H265 Encoder"; + info.scaling_settings = VideoEncoder::ScalingSettings::kOff; + info.is_hardware_accelerated = true; + info.supports_simulcast = false; + info.preferred_pixel_formats = {VideoFrameBuffer::Type::kI420}; + return info; +} + +void NvidiaH265EncoderImpl::SetRates( + const RateControlParameters& parameters) { + if (!encoder_) { + RTC_LOG(LS_WARNING) << "SetRates() while uninitialized."; + return; + } + + if (parameters.framerate_fps < 1.0) { + RTC_LOG(LS_WARNING) << "Invalid frame rate: " << parameters.framerate_fps; + return; + } + + if (parameters.bitrate.get_sum_bps() == 0) { + configuration_.SetStreamState(false); + return; + } + + codec_.maxFramerate = static_cast(parameters.framerate_fps); + codec_.maxBitrate = parameters.bitrate.GetSpatialLayerSum(0); + + configuration_.target_bps = parameters.bitrate.GetSpatialLayerSum(0); + configuration_.max_frame_rate = parameters.framerate_fps; + + if (configuration_.target_bps) { + configuration_.SetStreamState(true); + } else { + configuration_.SetStreamState(false); + } +} + +void NvidiaH265EncoderImpl::LayerConfig::SetStreamState(bool send_stream) { + if (send_stream && !sending) { + key_frame_request = true; + } + sending = send_stream; +} + +} // namespace webrtc + + diff --git a/webrtc-sys/src/nvidia/h265_encoder_impl.h b/webrtc-sys/src/nvidia/h265_encoder_impl.h new file mode 100644 index 000000000..63abe975c --- /dev/null +++ b/webrtc-sys/src/nvidia/h265_encoder_impl.h @@ -0,0 +1,94 @@ +#ifndef WEBRTC_NVIDIA_H265_ENCODER_IMPL_H_ +#define WEBRTC_NVIDIA_H265_ENCODER_IMPL_H_ + +#include + +#include +#include + +#include "NvEncoder/NvEncoder.h" +#include "NvEncoder/NvEncoderCuda.h" + +#include "absl/container/inlined_vector.h" +#include "api/environment/environment.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/transport/rtp/dependency_descriptor.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_codec_constants.h" +#include "api/video_codecs/scalability_mode.h" +#include "api/video_codecs/video_encoder.h" + +namespace webrtc { + +class NvidiaH265EncoderImpl : public VideoEncoder { + public: + struct LayerConfig { + int simulcast_idx = 0; + int width = -1; + int height = -1; + bool sending = true; + bool key_frame_request = false; + float max_frame_rate = 0; + uint32_t target_bps = 0; + uint32_t max_bps = 0; + bool frame_dropping_on = false; + int key_frame_interval = 0; + int num_temporal_layers = 1; + + void SetStreamState(bool send_stream); + }; + + public: + NvidiaH265EncoderImpl(const webrtc::Environment& env, + CUcontext context, + CUmemorytype memory_type, + NV_ENC_BUFFER_FORMAT nv_format, + const SdpVideoFormat& format); + ~NvidiaH265EncoderImpl() override; + + int32_t InitEncode(const VideoCodec* codec_settings, + const Settings& settings) override; + + int32_t RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) override; + + int32_t Release() override; + + int32_t Encode(const VideoFrame& frame, + const std::vector* frame_types) override; + + void SetRates(const RateControlParameters& rc_parameters) override; + + EncoderInfo GetEncoderInfo() const override; + + private: + int32_t ProcessEncodedFrame(std::vector& packet, + const ::webrtc::VideoFrame& inputFrame); + private: + const webrtc::Environment& env_; + EncodedImageCallback* encoded_image_callback_ = nullptr; + + std::unique_ptr encoder_; + CUcontext cu_context_; + CUmemorytype cu_memory_type_; + CUarray cu_scaled_array_; + NV_ENC_BUFFER_FORMAT nv_format_; + NV_ENC_INITIALIZE_PARAMS nv_initialize_params_; + NV_ENC_CONFIG nv_encode_config_; + + LayerConfig configuration_; + EncodedImage encoded_image_; + VideoCodec codec_; + void ReportInit(); + void ReportError(); + bool has_reported_init_ = false; + bool has_reported_error_ = false; + const SdpVideoFormat format_; + bool current_encoding_is_keyframe_ = false; +}; + +} // namespace webrtc + +#endif // WEBRTC_NVIDIA_H265_ENCODER_IMPL_H_ + + diff --git a/webrtc-sys/src/nvidia/nvidia_decoder_factory.cpp b/webrtc-sys/src/nvidia/nvidia_decoder_factory.cpp index 00fde1026..40c0a198d 100644 --- a/webrtc-sys/src/nvidia/nvidia_decoder_factory.cpp +++ b/webrtc-sys/src/nvidia/nvidia_decoder_factory.cpp @@ -6,6 +6,7 @@ #include "cuda_context.h" #include "h264_decoder_impl.h" +#include "h265_decoder_impl.h" #include "rtc_base/logging.h" namespace webrtc { @@ -53,6 +54,8 @@ std::vector SupportedNvDecoderCodecs(CUcontext context) { webrtc::H264Level::kLevel5_1, "1"), CreateH264Format(webrtc::H264Profile::kProfileMain, webrtc::H264Level::kLevel5_1, "1"), + SdpVideoFormat("H265"), + SdpVideoFormat("HEVC"), }; } @@ -92,7 +95,7 @@ std::unique_ptr NvidiaVideoDecoderFactory::Create( // Check if the requested format is supported. for (const auto& supported_format : supported_formats_) { if (format.IsSameCodec(supported_format)) { - // If the format is supported, create and return the encoder. + // If the format is supported, create and return the decoder. if (!cu_context_) { cu_context_ = livekit::CudaContext::GetInstance(); if (!cu_context_->Initialize()) { @@ -100,7 +103,14 @@ std::unique_ptr NvidiaVideoDecoderFactory::Create( return nullptr; } } - return std::make_unique(cu_context_->GetContext()); + if (format.name == "H264") { + RTC_LOG(LS_INFO) << "Using NVIDIA HW decoder (NVDEC) for H264"; + return std::make_unique(cu_context_->GetContext()); + } + if (format.name == "H265" || format.name == "HEVC") { + RTC_LOG(LS_INFO) << "Using NVIDIA HW decoder (NVDEC) for H265/HEVC"; + return std::make_unique(cu_context_->GetContext()); + } } } return nullptr; diff --git a/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp b/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp index a30c96f9f..7c8d0d95c 100644 --- a/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp +++ b/webrtc-sys/src/nvidia/nvidia_encoder_factory.cpp @@ -4,6 +4,7 @@ #include "cuda_context.h" #include "h264_encoder_impl.h" +#include "h265_encoder_impl.h" #include "rtc_base/logging.h" namespace webrtc { @@ -16,6 +17,11 @@ NvidiaVideoEncoderFactory::NvidiaVideoEncoderFactory() { }; supported_formats_.push_back(SdpVideoFormat("H264", baselineParameters)); + // Advertise HEVC/H265 with default parameters. + supported_formats_.push_back(SdpVideoFormat("H265")); + // Some stacks use 'HEVC' name. + supported_formats_.push_back(SdpVideoFormat("HEVC")); + /*std::map highParameters = { {"profile-level-id", "4d0032"}, {"level-asymmetry-allowed", "1"}, @@ -44,7 +50,6 @@ std::unique_ptr NvidiaVideoEncoderFactory::Create( // Check if the requested format is supported. for (const auto& supported_format : supported_formats_) { if (format.IsSameCodec(supported_format)) { - // If the format is supported, create and return the encoder. if (!cu_context_) { cu_context_ = livekit::CudaContext::GetInstance(); if (!cu_context_->Initialize()) { @@ -52,9 +57,20 @@ std::unique_ptr NvidiaVideoEncoderFactory::Create( return nullptr; } } - return std::make_unique( - env, cu_context_->GetContext(), CU_MEMORYTYPE_DEVICE, - NV_ENC_BUFFER_FORMAT_IYUV, format); + + if (format.name == "H264") { + RTC_LOG(LS_INFO) << "Using NVIDIA HW encoder (NVENC) for H264"; + return std::make_unique( + env, cu_context_->GetContext(), CU_MEMORYTYPE_DEVICE, + NV_ENC_BUFFER_FORMAT_IYUV, format); + } + + if (format.name == "H265" || format.name == "HEVC") { + RTC_LOG(LS_INFO) << "Using NVIDIA HW encoder (NVENC) for H265/HEVC"; + return std::make_unique( + env, cu_context_->GetContext(), CU_MEMORYTYPE_DEVICE, + NV_ENC_BUFFER_FORMAT_IYUV, format); + } } } return nullptr;