Pass all codecs instead of selected one to the rtp_receiver and rtp_sender

lib/ex_webrtc/rtp_receiver.ex

@@ -61,9 +61,11 @@ defmodule ExWebRTC.RTPReceiver do
   end
 
   @doc false
-  @spec new(MediaStreamTrack.t(), RTPCodecParameters.t() | nil, [Extmap.t()], [atom()]) ::
-          receiver()
-  def new(track, codec, rtp_hdr_exts, features) do
+  @spec new(MediaStreamTrack.t(), [RTPCodecParameters.t()], [Extmap.t()], [atom()]) :: receiver()
+  def new(track, codecs, rtp_hdr_exts, features) do
+    {_rtx_codecs, media_codecs} = Utils.split_rtx_codecs(codecs)
+    codec = List.first(media_codecs)
+
     # layer `nil` is for the packets without RID/ no simulcast
     %{
       id: Utils.generate_id(),
@@ -77,8 +79,10 @@ defmodule ExWebRTC.RTPReceiver do
   end
 
   @doc false
-  @spec update(receiver(), RTPCodecParameters.t() | nil, [Extmap.t()], [String.t()]) :: receiver()
-  def update(receiver, codec, rtp_hdr_exts, stream_ids) do
+  @spec update(receiver(), [RTPCodecParameters.t()], [Extmap.t()], [String.t()]) :: receiver()
+  def update(receiver, codecs, rtp_hdr_exts, stream_ids) do
+    {_rtx_codecs, media_codecs} = Utils.split_rtx_codecs(codecs)
+    codec = List.first(media_codecs)
     simulcast_demuxer = SimulcastDemuxer.update(receiver.simulcast_demuxer, rtp_hdr_exts)
     track = %MediaStreamTrack{receiver.track | streams: stream_ids}
 

lib/ex_webrtc/rtp_sender.ex

@@ -17,6 +17,7 @@ defmodule ExWebRTC.RTPSender do
           id: id(),
           track: MediaStreamTrack.t() | nil,
           codec: RTPCodecParameters.t() | nil,
+          codecs: [RTPCodecParameters.t()],
           rtp_hdr_exts: %{Extmap.extension_id() => Extmap.t()},
           mid: String.t() | nil,
           pt: non_neg_integer() | nil,
@@ -67,17 +68,23 @@ defmodule ExWebRTC.RTPSender do
   @doc false
   @spec new(
           MediaStreamTrack.t() | nil,
-          RTPCodecParameters.t() | nil,
-          RTPCodecParameters.t() | nil,
+          [RTPCodecParameters.t()],
           [Extmap.t()],
           String.t() | nil,
           non_neg_integer(),
           non_neg_integer(),
           [atom()]
         ) :: sender()
-  def new(track, codec, rtx_codec, rtp_hdr_exts, mid, ssrc, rtx_ssrc, features) do
+  def new(track, codecs, rtp_hdr_exts, mid, ssrc, rtx_ssrc, features) do
     # convert to a map to be able to find extension id using extension uri
     rtp_hdr_exts = Map.new(rtp_hdr_exts, fn extmap -> {extmap.uri, extmap} end)
+
+    # We always only take one codec to avoid ambiguity when assigning payload type for RTP packets.
+    # In other case, if PeerConnection negotiated multiple codecs,
+    # user would have to pass RTP codec when sending RTP packets,
+    # or assign payload type on their own.
+    {codec, rtx_codec} = get_default_codec(codecs)
+
     # TODO: handle cases when codec == nil (no valid codecs after negotiation)
     pt = if codec != nil, do: codec.payload_type, else: nil
     rtx_pt = if rtx_codec != nil, do: rtx_codec.payload_type, else: nil
@@ -86,6 +93,7 @@ defmodule ExWebRTC.RTPSender do
       id: Utils.generate_id(),
       track: track,
       codec: codec,
+      codecs: codecs,
       rtp_hdr_exts: rtp_hdr_exts,
       pt: pt,
       rtx_pt: rtx_pt,
@@ -107,11 +115,12 @@ defmodule ExWebRTC.RTPSender do
   end
 
   @doc false
-  @spec update(sender(), String.t(), RTPCodecParameters.t() | nil, RTPCodecParameters.t() | nil, [
-          Extmap.t()
-        ]) :: sender()
-  def update(sender, mid, codec, rtx_codec, rtp_hdr_exts) do
+  @spec update(sender(), String.t(), [RTPCodecParameters.t()], [Extmap.t()]) :: sender()
+  def update(sender, mid, codecs, rtp_hdr_exts) do
     if sender.mid != nil and mid != sender.mid, do: raise(ArgumentError)
+
+    {codec, rtx_codec} = get_default_codec(codecs)
+
     # convert to a map to be able to find extension id using extension uri
     rtp_hdr_exts = Map.new(rtp_hdr_exts, fn extmap -> {extmap.uri, extmap} end)
     # TODO: handle cases when codec == nil (no valid codecs after negotiation)
@@ -127,6 +136,7 @@ defmodule ExWebRTC.RTPSender do
       sender
       | mid: mid,
         codec: codec,
+        codecs: codecs,
         rtp_hdr_exts: rtp_hdr_exts,
         pt: pt,
         rtx_pt: rtx_pt,
@@ -326,4 +336,20 @@ defmodule ExWebRTC.RTPSender do
       pli_count: sender.pli_count
     }
   end
+
+  defp get_default_codec(codecs) do
+    {rtx_codecs, media_codecs} = Utils.split_rtx_codecs(codecs)
+
+    case List.first(media_codecs) do
+      nil ->
+        {nil, nil}
+
+      codec ->
+        {codec, find_associated_rtx_codec(rtx_codecs, codec)}
+    end
+  end
+
+  defp find_associated_rtx_codec(codecs, codec) do
+    Enum.find(codecs, &(&1.sdp_fmtp_line && &1.sdp_fmtp_line.apt == codec.payload_type))
+  end
 end

lib/ex_webrtc/rtp_transceiver.ex

@@ -123,12 +123,6 @@ defmodule ExWebRTC.RTPTransceiver do
         :video -> {config.video_extensions, config.video_codecs}
       end
 
-    # When we create sendonly or sendrecv transceiver, we always only take one codec
-    # to avoid ambiguity when assigning payload type for RTP packets in RTPSender.
-    # In other case, if PeerConnection negotiated multiple codecs,
-    # user would have to pass RTP codec when sending RTP packets,
-    # or assign payload type on their own.
-    {codec, codec_rtx} = get_default_codec(codecs)
     track = MediaStreamTrack.new(kind)
 
     id = Utils.generate_id()
@@ -141,13 +135,12 @@ defmodule ExWebRTC.RTPTransceiver do
       send(self(), {:send_nacks, id})
     end
 
-    receiver = RTPReceiver.new(track, codec, header_extensions, config.features)
+    receiver = RTPReceiver.new(track, codecs, header_extensions, config.features)
 
     sender =
       RTPSender.new(
         sender_track,
-        codec,
-        codec_rtx,
+        codecs,
         header_extensions,
         nil,
         options[:ssrc],
@@ -196,7 +189,6 @@ defmodule ExWebRTC.RTPTransceiver do
     {:mid, mid} = ExSDP.get_attribute(mline, :mid)
 
     track = MediaStreamTrack.from_mline(mline)
-    {codec, codec_rtx} = get_default_codec(codecs)
 
     id = Utils.generate_id()
 
@@ -208,13 +200,12 @@ defmodule ExWebRTC.RTPTransceiver do
       send(self(), {:send_nacks, id})
     end
 
-    receiver = RTPReceiver.new(track, codec, header_extensions, config.features)
+    receiver = RTPReceiver.new(track, codecs, header_extensions, config.features)
 
     sender =
       RTPSender.new(
         nil,
-        codec,
-        codec_rtx,
+        codecs,
         header_extensions,
         mid,
         ssrc,
@@ -265,11 +256,10 @@ defmodule ExWebRTC.RTPTransceiver do
 
     codecs = Configuration.intersect_codecs(config, mline)
     header_extensions = Configuration.intersect_extensions(config, mline)
-    {codec, codec_rtx} = get_default_codec(codecs)
     stream_ids = SDPUtils.get_stream_ids(mline)
 
-    receiver = RTPReceiver.update(transceiver.receiver, codec, header_extensions, stream_ids)
-    sender = RTPSender.update(transceiver.sender, mid, codec, codec_rtx, header_extensions)
+    receiver = RTPReceiver.update(transceiver.receiver, codecs, header_extensions, stream_ids)
+    sender = RTPSender.update(transceiver.sender, mid, codecs, header_extensions)
 
     %{
       transceiver
@@ -600,17 +590,4 @@ defmodule ExWebRTC.RTPTransceiver do
     factor = :rand.uniform() + 0.5
     trunc(factor * @report_interval)
   end
-
-  defp get_default_codec(codecs) do
-    {rtxs, codecs} = Enum.split_with(codecs, &String.ends_with?(&1.mime_type, "/rtx"))
-
-    case List.first(codecs) do
-      nil ->
-        {nil, nil}
-
-      codec ->
-        rtx = Enum.find(rtxs, &(&1.sdp_fmtp_line.apt == codec.payload_type))
-        {codec, rtx}
-    end
-  end
 end

lib/ex_webrtc/utils.ex

@@ -1,5 +1,6 @@
 defmodule ExWebRTC.Utils do
   @moduledoc false
+  alias ExWebRTC.RTPCodecParameters
 
   @spec hex_dump(binary()) :: String.t()
   def hex_dump(binary) do
@@ -17,4 +18,10 @@ defmodule ExWebRTC.Utils do
   @spec to_int(boolean()) :: 0 | 1
   def to_int(false), do: 0
   def to_int(true), do: 1
+
+  @spec split_rtx_codecs([RTPCodecParameters.t()]) ::
+          {[RTPCodecParameters.t()], [RTPCodecParameters.t()]}
+  def split_rtx_codecs(codecs) do
+    Enum.split_with(codecs, &String.ends_with?(&1.mime_type, "/rtx"))
+  end
 end

test/ex_webrtc/rtp_receiver_test.exs

@@ -11,7 +11,7 @@ defmodule ExWebRTC.RTPReceiverTest do
     payload = <<1, 2, 3>>
 
     track = MediaStreamTrack.new(:audio)
-    receiver = RTPReceiver.new(track, @codec, [], [])
+    receiver = RTPReceiver.new(track, [@codec], [], [])
 
     assert [] == RTPReceiver.get_stats(receiver, timestamp)
 

test/ex_webrtc/rtp_sender_test.exs

@@ -30,7 +30,7 @@ defmodule ExWebRTC.RTPSenderTest do
   setup do
     track = MediaStreamTrack.new(:video)
 
-    sender = RTPSender.new(track, @codec, nil, @rtp_hdr_exts, "1", @ssrc, @rtx_ssrc, [])
+    sender = RTPSender.new(track, [@codec], @rtp_hdr_exts, "1", @ssrc, @rtx_ssrc, [])
 
     %{sender: sender}
   end
@@ -66,7 +66,7 @@ defmodule ExWebRTC.RTPSenderTest do
       stream_id = MediaStreamTrack.generate_stream_id()
       track = MediaStreamTrack.new(:video, [stream_id])
 
-      sender = RTPSender.new(track, @codec, nil, @rtp_hdr_exts, "1", @ssrc, @rtx_ssrc, [])
+      sender = RTPSender.new(track, [@codec], @rtp_hdr_exts, "1", @ssrc, @rtx_ssrc, [])
 
       assert [
                %ExSDP.Attribute.MSID{id: ^stream_id, app_data: nil},
@@ -78,7 +78,8 @@ defmodule ExWebRTC.RTPSenderTest do
       stream_id = MediaStreamTrack.generate_stream_id()
       track = MediaStreamTrack.new(:video, [stream_id])
 
-      sender = RTPSender.new(track, @codec, @rtx_codec, @rtp_hdr_exts, "1", @ssrc, @rtx_ssrc, [])
+      sender =
+        RTPSender.new(track, [@codec, @rtx_codec], @rtp_hdr_exts, "1", @ssrc, @rtx_ssrc, [])
 
       assert [
                %ExSDP.Attribute.MSID{id: ^stream_id, app_data: nil},
@@ -91,7 +92,8 @@ defmodule ExWebRTC.RTPSenderTest do
     test "without media stream" do
       track = MediaStreamTrack.new(:video)
 
-      sender = RTPSender.new(track, @codec, @rtx_codec, @rtp_hdr_exts, "1", @ssrc, @rtx_ssrc, [])
+      sender =
+        RTPSender.new(track, [@codec, @rtx_codec], @rtp_hdr_exts, "1", @ssrc, @rtx_ssrc, [])
 
       assert [
                %ExSDP.Attribute.MSID{id: "-", app_data: nil},
@@ -107,7 +109,8 @@ defmodule ExWebRTC.RTPSenderTest do
 
       track = MediaStreamTrack.new(:video, [s1_id, s2_id])
 
-      sender = RTPSender.new(track, @codec, @rtx_codec, @rtp_hdr_exts, "1", @ssrc, @rtx_ssrc, [])
+      sender =
+        RTPSender.new(track, [@codec, @rtx_codec], @rtp_hdr_exts, "1", @ssrc, @rtx_ssrc, [])
 
       assert [
                %ExSDP.Attribute.MSID{id: ^s1_id, app_data: nil},