Add set_sender_codec

Author: mickel8

lib/ex_webrtc/peer_connection.ex

@@ -9,6 +9,7 @@ defmodule ExWebRTC.PeerConnection do
 
   require Logger
 
+  alias ExWebRTC.RTPCodecParameters
   alias __MODULE__.{Configuration, Demuxer, TWCCRecorder}
 
   alias ExWebRTC.{
@@ -152,9 +153,61 @@ defmodule ExWebRTC.PeerConnection do
     GenServer.call(peer_connection, :get_configuration)
   end
 
+  @doc """
+  Sets the codec that will be used for sending RTP packets.
+
+  `send_rtp/4` overrides some of the RTP packet fields.
+  In particular, when multiple codecs are negotiated, `send_rtp/4` will use
+  payload type of the most preffered by the remote side codec (i.e. the first
+  one from the list of codecs in the remote description).
+
+  Use this function if you want to select, which codec (hence payload type)
+  should be used for sending.
+
+  Once the first RTP packet is sent (via `send_rtp/4`), `set_sender_codec/3`
+  can only be called with a codec with the clock rate.
+
+  Although very unlikely, keep in mind that after renegotiation,
+  the selected codec may no longer be supported by the remote side and you might
+  need to call this function again, passing a new codec.
+
+  To check available codecs you can use `get_transceivers/1`:
+
+  ```
+  {:ok, pc} = PeerConnection.start_link()
+  {:ok, rtp_sender} = PeerConnection.add_track(MediaStreamTrack.new(:video))
+
+  tr =
+    pc
+    |> PeerConnection.get_transceivers()
+    |> Enum.find(fn tr -> tr.sender.id == rtp_sender.id end)
+
+  dbg(tr.codecs) # list of supported codecs both for sending and receiving
+
+  # e.g. always prefer h264 over vp8
+  h264 = Enum.find(tr.codecs, fn codec -> codec.mime_type == "video/H264" end)
+  vp8 = Enum.find(tr.codecs, fn codec -> codec.mime_type == "video/VP8" end)
+
+  :ok = PeerConnection.set_sender_codec(pc, rtp_sender.id, h264 || vp8)
+  ```
+  """
+  @spec set_sender_codec(peer_connection(), RTPSender.id(), RTPCodecParameters.t()) ::
+          :ok | {:error, term()}
+  def set_sender_codec(peer_connection, sender_id, codec) do
+    GenServer.call(peer_connection, {:set_sender_codec, sender_id, codec})
+  end
+
   @doc """
   Sends an RTP packet to the remote peer using the track specified by the `track_id`.
 
+  The following fields of the RTP packet will be overwritten by this function:
+  * payload type
+  * ssrc
+  * rtp header extensions
+
+  If you negotiated multiple codecs (hence payload types) and you want to choose,
+  which one should be used, see `set_sender_codec/3`.
+
   Options:
     * `rtx?` - send the packet as if it was retransmitted (use SSRC and payload type specific to RTX)
   """
@@ -576,6 +629,31 @@ defmodule ExWebRTC.PeerConnection do
     {:reply, state.config, state}
   end
 
+  @impl true
+  def handle_call({:set_sender_codec, sender_id, codec}, _from, state) do
+    state.transceivers
+    |> Enum.with_index()
+    |> Enum.find(fn {tr, _idx} -> tr.sender.id == sender_id end)
+    |> case do
+      {tr, idx} when tr.direction in [:sendrecv, :sendonly] ->
+        case RTPTransceiver.set_sender_codec(tr, codec) do
+          {:ok, tr} ->
+            transceivers = List.replace_at(state.transceivers, idx, tr)
+            state = %{state | transceivers: transceivers}
+            {:reply, :ok, state}
+
+          {:error, _reason} = error ->
+            {:reply, error, state}
+        end
+
+      {_tr, _idx} ->
+        {:reply, {:error, :invalid_transceiver_direction}, state}
+
+      nil ->
+        {:reply, {:error, :invalid_sender_id}, state}
+    end
+  end
+
   @impl true
   def handle_call(:get_connection_state, _from, state) do
     {:reply, state.conn_state, state}
@@ -1135,7 +1213,8 @@ defmodule ExWebRTC.PeerConnection do
           end
 
         {packet, tr} = RTPTransceiver.send_packet(tr, packet, rtx?)
-        :ok = DTLSTransport.send_rtp(state.dtls_transport, packet)
+
+        if packet != <<>>, do: :ok = DTLSTransport.send_rtp(state.dtls_transport, packet)
 
         transceivers = List.replace_at(state.transceivers, idx, tr)
         state = %{state | transceivers: transceivers}

lib/ex_webrtc/rtp_sender.ex

@@ -2,6 +2,7 @@ defmodule ExWebRTC.RTPSender do
   @moduledoc """
   Implementation of the [RTCRtpSender](https://www.w3.org/TR/webrtc/#rtcrtpsender-interface).
   """
+  require Logger
 
   alias ExRTCP.Packet.{TransportFeedback.NACK, PayloadFeedback.PLI}
   alias ExWebRTC.{MediaStreamTrack, RTPCodecParameters, Utils}
@@ -17,6 +18,7 @@ defmodule ExWebRTC.RTPSender do
           id: id(),
           track: MediaStreamTrack.t() | nil,
           codec: RTPCodecParameters.t() | nil,
+          rtx_codec: RTPCodecParameters.t() | nil,
           codecs: [RTPCodecParameters.t()],
           rtp_hdr_exts: %{Extmap.extension_id() => Extmap.t()},
           mid: String.t() | nil,
@@ -93,6 +95,7 @@ defmodule ExWebRTC.RTPSender do
       id: Utils.generate_id(),
       track: track,
       codec: codec,
+      rtx_codec: rtx_codec,
       codecs: codecs,
       rtp_hdr_exts: rtp_hdr_exts,
       pt: pt,
@@ -109,7 +112,7 @@ defmodule ExWebRTC.RTPSender do
       pli_count: 0,
       reports?: :rtcp_reports in features,
       outbound_rtx?: :outbound_rtx in features,
-      report_recorder: %ReportRecorder{clock_rate: codec && codec.clock_rate},
+      report_recorder: %ReportRecorder{},
       nack_responder: %NACKResponder{}
     }
   end
@@ -118,32 +121,56 @@ defmodule ExWebRTC.RTPSender do
   @spec update(sender(), String.t(), [RTPCodecParameters.t()], [Extmap.t()]) :: sender()
   def update(sender, mid, codecs, rtp_hdr_exts) do
     if sender.mid != nil and mid != sender.mid, do: raise(ArgumentError)
-
-    {codec, rtx_codec} = get_default_codec(codecs)
-
     # convert to a map to be able to find extension id using extension uri
     rtp_hdr_exts = Map.new(rtp_hdr_exts, fn extmap -> {extmap.uri, extmap} end)
+
+    # Keep already selected codec if it is still supported.
+    # Otherwise, clear it and wait until user sets it again.
+    codec = if sender.codec in codecs, do: sender.codec, else: nil
+    rtx_codec = codec && find_associated_rtx_codec(codecs, codec)
+
+    log_codec_change(sender, codec, codecs)
+    log_rtx_codec_change(sender, rtx_codec, codecs)
+
     # TODO: handle cases when codec == nil (no valid codecs after negotiation)
     pt = if codec != nil, do: codec.payload_type, else: nil
     rtx_pt = if rtx_codec != nil, do: rtx_codec.payload_type, else: nil
 
-    report_recorder = %ReportRecorder{
-      sender.report_recorder
-      | clock_rate: codec && codec.clock_rate
-    }
-
     %{
       sender
       | mid: mid,
         codec: codec,
+        rtx_codec: rtx_codec,
         codecs: codecs,
         rtp_hdr_exts: rtp_hdr_exts,
         pt: pt,
-        rtx_pt: rtx_pt,
-        report_recorder: report_recorder
+        rtx_pt: rtx_pt
     }
   end
 
+  defp log_codec_change(%{codec: codec} = sender, nil, neg_codecs) when codec != nil do
+    Logger.debug("""
+    Unselecting RTP sender codec as it is no longer supported by the remote side.
+    Call set_sender_codec again passing supported codec.
+    Codec: #{inspect(sender.codec)}
+    Currently negotiated codecs: #{inspect(neg_codecs)}
+    """)
+  end
+
+  defp log_codec_change(_sender, _codec, _neg_codecs), do: :ok
+
+  defp log_rtx_codec_change(%{rtx_codec: rtx_codec} = sender, nil, neg_codecs)
+       when rtx_codec != nil do
+    Logger.debug("""
+    Unselecting RTP sender codec as it is no longer supported by the remote side.
+    Call set_sender_codec again passing supported codec.
+    Codec: #{inspect(sender.codec)}
+    Currently negotiated codecs: #{inspect(neg_codecs)}
+    """)
+  end
+
+  defp log_rtx_codec_change(_sender, _rtx_codec, _neg_codecs), do: :ok
+
   @spec get_mline_attrs(sender()) :: [ExSDP.Attribute.t()]
   def get_mline_attrs(sender) do
     # Don't include track id. See RFC 8829 sec. 5.2.1
@@ -221,9 +248,53 @@ defmodule ExWebRTC.RTPSender do
     [fid | ssrc_attrs]
   end
 
+  @doc false
+  @spec set_codec(sender(), RTPCodecParameters.t()) :: {:ok, sender()} | {:error, term()}
+  def set_codec(sender, codec) do
+    if not rtx?(codec) and supported?(sender, codec) and same_clock_rate?(sender, codec) do
+      rtx_codec = find_associated_rtx_codec(sender.codecs, codec)
+      sender = %{sender | codec: codec, rtx_codec: rtx_codec}
+      {:ok, sender}
+    else
+      {:error, :invalid_codec}
+    end
+  end
+
+  defp rtx?(codec), do: String.ends_with?(codec.mime_type, "rtx")
+  defp supported?(sender, codec), do: codec in sender.codecs
+
+  # As long as report recorder is not initialized i.e. we have not send any RTP packet
+  # allow for codec changes. Once we start sending RTP packet, require the same clock rate.
+  defp same_clock_rate?(%{report_recorder: %{clock_rate: nil}}, _codec), do: true
+  defp same_clock_rate?(sender, codec), do: sender.report_recorder.clock_rate == codec.clock_rate
+
   @doc false
   @spec send_packet(sender(), ExRTP.Packet.t(), boolean()) :: {binary(), sender()}
+  def send_packet(%{rtx_codec: nil} = sender, _packet, true) do
+    Logger.warning("Tried to retransmit packet but there is no selected RTX codec. Ignoring.")
+    {<<>>, sender}
+  end
+
+  def send_packet(%{codec: nil} = sender, _packet, false) do
+    Logger.warning("Tried to send packet but there is no selected codec. Ignoring.")
+    {<<>>, sender}
+  end
+
+  def send_packet(%{packets_sent: 0}, _packet, true) do
+    raise "Tried to retransmit packet without sending any real RTP packet. This should never happen."
+  end
+
+  def send_packet(%{packets_sent: 0} = sender, packet, false) do
+    recorder = ReportRecorder.init(sender.report_recorder, sender.codec.clock_rate, sender.ssrc)
+    sender = %{sender | report_recorder: recorder}
+    do_send_packet(sender, packet, false)
+  end
+
   def send_packet(sender, packet, rtx?) do
+    do_send_packet(sender, packet, rtx?)
+  end
+
+  def do_send_packet(sender, packet, rtx?) do
     {pt, ssrc} =
       if rtx? do
         {sender.rtx_pt, sender.rtx_ssrc}

lib/ex_webrtc/rtp_sender/report_recorder.ex

@@ -29,6 +29,14 @@ defmodule ExWebRTC.RTPSender.ReportRecorder do
             packet_count: 0,
             octet_count: 0
 
+  @spec init(t(), non_neg_integer(), non_neg_integer()) :: t()
+  def init(%{clock_rate: nil, sender_ssrc: nil}, clock_rate, sender_ssrc) do
+    %__MODULE__{clock_rate: clock_rate, sender_ssrc: sender_ssrc}
+  end
+
+  def init(_recorder, _clock_rate, _sender_ssrc),
+    do: raise("Tried to re-initialize ReportRecorder")
+
   @doc """
   Records incoming RTP packet.
 

lib/ex_webrtc/rtp_transceiver.ex

@@ -277,6 +277,16 @@ defmodule ExWebRTC.RTPTransceiver do
     %{transceiver | direction: direction}
   end
 
+  @doc false
+  @spec set_sender_codec(transceiver(), RTPCodecParameters.t()) ::
+          {:ok, transceiver()} | {:error, term()}
+  def set_sender_codec(transceiver, codec) do
+    case RTPSender.set_codec(transceiver.sender, codec) do
+      {:ok, sender} -> {:ok, %{transceiver | sender: sender}}
+      {:error, _reason} = error -> error
+    end
+  end
+
   @doc false
   @spec can_add_track?(transceiver(), kind()) :: boolean()
   def can_add_track?(transceiver, kind) do
@@ -367,18 +377,22 @@ defmodule ExWebRTC.RTPTransceiver do
   @doc false
   @spec send_packet(transceiver(), ExRTP.Packet.t(), boolean()) :: {binary(), transceiver()}
   def send_packet(transceiver, packet, rtx?) do
-    {packet, sender} = RTPSender.send_packet(transceiver.sender, packet, rtx?)
+    case RTPSender.send_packet(transceiver.sender, packet, rtx?) do
+      {<<>>, sender} ->
+        {<<>>, %{transceiver | sender: sender}}
 
-    receiver =
-      if rtx? do
-        transceiver.receiver
-      else
-        RTPReceiver.update_sender_ssrc(transceiver.receiver, sender.ssrc)
-      end
+      {packet, sender} ->
+        receiver =
+          if rtx? do
+            transceiver.receiver
+          else
+            RTPReceiver.update_sender_ssrc(transceiver.receiver, sender.ssrc)
+          end
 
-    transceiver = %{transceiver | sender: sender, receiver: receiver}
+        transceiver = %{transceiver | sender: sender, receiver: receiver}
 
-    {packet, transceiver}
+        {packet, transceiver}
+    end
   end
 
   @doc false

test/ex_webrtc/peer_connection_test.exs

@@ -219,6 +219,23 @@ defmodule ExWebRTC.PeerConnectionTest do
     assert_receive {:ex_webrtc, _pid, {:connection_state_change, :new}}
   end
 
+  test "set_sender_codec" do
+    {:ok, pid} = PeerConnection.start_link()
+    {:ok, tr} = PeerConnection.add_transceiver(pid, :video)
+
+    {rtx_codecs, media_codecs} = Utils.split_rtx_codecs(tr.codecs)
+
+    assert :ok = PeerConnection.set_sender_codec(pid, tr.sender.id, List.last(media_codecs))
+
+    assert {:error, :invalid_sender_id} =
+             PeerConnection.set_sender_codec(pid, "invalid_id", List.last(media_codecs))
+
+    :ok = PeerConnection.set_transceiver_direction(pid, tr.id, :recvonly)
+
+    assert {:error, :invalid_transceiver_direction} =
+             PeerConnection.set_sender_codec(pid, tr.sender.id, List.last(media_codecs))
+  end
+
   test "send_rtp/4" do
     {:ok, pc1} = PeerConnection.start_link()
     {:ok, pc2} = PeerConnection.start_link()

test/ex_webrtc/rtp_sender/report_recorder_test.exs

@@ -15,6 +15,15 @@ defmodule ExWebRTC.RTPSender.ReportRecorderTest do
   @ntp_offset 2_208_988_800
   @max_u32 0xFFFFFFFF
 
+  test "init/3" do
+    recorder = %ReportRecorder{}
+
+    %{clock_rate: 90_000, sender_ssrc: 1234} =
+      recorder = ReportRecorder.init(recorder, 90_000, 1234)
+
+    assert_raise RuntimeError, fn -> ReportRecorder.init(recorder, 90_000, 1234) end
+  end
+
   describe "record_packet/3" do
     test "keeps track of packet counts and sizes" do
       recorder =
@@ -78,7 +87,7 @@ defmodule ExWebRTC.RTPSender.ReportRecorderTest do
 
       native_in_sec = System.convert_time_unit(1, :second, :native)
       seconds = 89_934
-      # 1/8, so 0.001 in binary 
+      # 1/8, so 0.001 in binary
       frac = 0.125
 
       assert {:ok, report, _recorder} =