chore: Add a DA service unit test (#3423)

<!-- Replace this block with what this PR does and why. Describe what
you'd like reviewers to know, how you applied the engineering
principles, and any interesting tradeoffs made. Delete bullet points
below that don't apply, and update the changelog section as appropriate.
-->

<!-- Add short version of the JIRA ticket to the PR title (e.g. "feat:
new shiny feature [MTT-123]") -->
[MTT-11550](https://jira.unity3d.com/browse/MTT-11550)

## Changelog

- Added: A unit test against the da service 

## Testing and Documentation

- Includes unit tests.

<!-- Uncomment and mark items off with a * if this PR deprecates any
API:
### Deprecated API
- [ ] An `[Obsolete]` attribute was added along with a `(RemovedAfter
yyyy-mm-dd)` entry.
- [ ] An [api updater] was added.
- [ ] Deprecation of the API is explained in the CHANGELOG.
- [ ] The users can understand why this API was removed and what they
should use instead.
-->

## Backport

<!-- If this is a backport:
 - Add the following to the PR title: "\[Backport\] ..." .
 - Link to the original PR.
If this needs a backport - state this here
If a backport is not needed please provide the reason why.
If the "Backports" section is not present it will lead to a CI test
failure.
-->

No backport applicable - DA only

---------

Co-authored-by: NoelStephensUnity <[email protected]>

Author: EmandM

.yamato/desktop-standalone-tests.yml

@@ -4,7 +4,7 @@
 # DESCRIPTION--------------------------------------------------------------------------
   # This job is responsible for Desktop platform test validation.
   # Those tests cover both PlayMode and EditMode tests from package test assemblies.
-  
+
 # CONFIGURATION STRUCTURE--------------------------------------------------------------
   # Jobs are generated using nested loops (separate build phase and run phase). Worth noting that run phase uses the build as dependency:
     # 1. For all desktop platform (Windows, macOS, Ubuntu)
@@ -17,17 +17,17 @@
     # 1. Build Phase: Creates standalone players for desktop platforms
     # 2. Run Phase: Executes runtime tests on actual desktop devices
   # The Run phase uses build job as dependency
-  
+
   # Note: More of a Unity specific but test assemblies need to be included in the build phase command
   # Note: All builds can be made on x64 machines since those are compatible with ARM64 target devices
 
 # QUALITY THOUGHTS--------------------------------------------------------------------
   # TODO: consider adding all projects that have tests
   # To see where this job is included (in trigger job definitions) look into _triggers.yml file
-  
+
 #-----------------------------------------------------------------------------------
-  
-  
+
+
 # BUILD PHASE CONFIGURATION------------------------------------------------------------------------------------
 {% for project in projects.default -%}
 {% for platform in test_platforms.desktop -%}
@@ -58,10 +58,10 @@ desktop_standalone_build_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{
 {% endfor -%}
 {% endfor -%}
 {% endfor -%}
-  
-  
-  
-  
+
+
+
+
 # RUN PHASE CONFIGURATION------------------------------------------------------------------------------------
 {% for project in projects.default -%}
 {% for platform in test_platforms.desktop -%}
@@ -80,22 +80,25 @@ desktop_standalone_test_{{ project.name }}_{{ platform.name }}_{{ backend }}_{{
 # Set additional variables for running the echo server (This is needed ONLY for NGOv2.X because relates to Distributed Authority)
 {% if platform.name != "win" %} # Issues with win and mac are tracked in MTT-11606
   variables:
+    # The echo server is a "mock" server that is only used to test encoding/decoding of messages.
+    # It is used by the DistributedAuthorityCodecTests. These are tests that are built and maintained by the CMB service team.
     ECHO_SERVER_PORT: "7788"
-    # Set this to ensure the DA codec tests will fail if they cannot connect to the echo-server
-    # The default is to ignore the codec tests if the echo-server fails to connect
+    # Set this to ensure the DistributedAuthorityCodecTests will fail if they cannot connect to the echo server.
+    # The default is to ignore the codec tests if the echo server fails to connect
     ENSURE_CODEC_TESTS: "true"
+
+    # When USE_CMB_SERVICE is set to true, any C# tests configured to use the DA host will instead use the CMB service.
+    USE_CMB_SERVICE: "true"
+    # This is the port on which to run the full standalone CMB service.
+    # The port needs to be different from the echo server port as two processes cannot bind to same port.
+    CMB_SERVICE_PORT: "7799"
 {% endif %}
 
   commands:
 # If ubuntu, run rust echo server (This is needed ONLY for NGOv2.X because relates to Distributed Authority)
 {% if platform.name != "win" %} # Issues with win and mac are tracked in MTT-11606
-    - git clone https://github.com/Unity-Technologies/mps-common-multiplayer-backend.git
-    # Install rust
-    - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-    # Build the echo server
-    - cd ./mps-common-multiplayer-backend/runtime && $HOME/.cargo/bin/cargo build --example ngo_echo_server
-    # Run the echo server in the background - this will reuse the artifacts from the build
-    - cd ./mps-common-multiplayer-backend/runtime && $HOME/.cargo/bin/cargo run --example ngo_echo_server -- --port $ECHO_SERVER_PORT &
+    # run_cmb_service.sh builds and starts a release version of the full CMB service (along with the limited echo server)
+    - ./Tools/CI/run_cmb_service.sh -e $ECHO_SERVER_PORT -s $CMB_SERVICE_PORT
 {% endif %}
 
     - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if backend == "il2cpp" %} -c il2cpp {% endif %} {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models

Tools/CI/run_cmb_service.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+# DESCRIPTION--------------------------------------------------------------------------
+  # This bash file is used to build and run CMB service resources.
+  # These are resources relating to the distributed authority feature. The CMB service is built and maintained by the services team.
+
+# CONTENTS-----------------------------------------------------------------------------
+  # There are two resources that are built and run:
+    # 1. The echo server - this is a "mock" server that simply echoes back the messages it receives. It is used to test message serialization.
+      #  The echo server is used inside the DistributedAuthorityCodecTests. These are tests that are built and maintained by the CMB service team.
+    # 2. The standalone server - this is a release build of the full CMB service.
+
+# USAGE---------------------------------------------------------------------------------
+  # The script requires ports to be defined. This works via the following command:
+    # ./<path-to-script>/run_cmb_service.sh -e <echo-server-port> -s <cmb-service-port>
+
+  # Example usage:
+    # ./<path-to-script>/run_cmb_service.sh -e 7788 -s 7799
+
+  # This script is currently used in the desktop-standalone-tests yamato job.
+
+# TECHNICAL CONSIDERATIONS---------------------------------------------------------------
+  # This is a bash script and so needs to be run on a Unix based system.
+    # - It runs on mac and ubuntu bokken machines. It will not run on windows bokken machines.
+
+  # This script starts processes running in the background. All logs are still written to stdout and will be seen in the logs of the job.
+  # Running in the background means that the processes will continue to run after the script exits.
+    # This is not a concern for bokken machines as all processes are killed when the machine is shut down.
+
+# QUALITY THOUGHTS--------------------------------------------------------------------
+  # Currently this script simply uses the head of the cmb service repo.
+  # We might want to consider using the latest released version in the future.
+
+#-------------------------------------------------------------------------------------
+
+# Define error message if ports are not defined
+ERROR="Error: Expected ports to be defined! Example script usage:"
+EXAMPLE="run_cmb_service.sh -e <echo-server-port> -s <cmb-service-port>"
+
+# get arguments passed to the script
+while getopts 'e:s:' flag; do
+case "${flag}" in
+  e) echo_port="${OPTARG}" ;;
+  s) service_port="${OPTARG}" ;;
+  *) printf "%s\n" "$ERROR" "$EXAMPLE"
+      exit 1 ;;
+  esac
+done
+
+# ensure arguments were passed and the ports are defined
+if [ -z "$echo_port" ] || [ -z "$service_port" ]; then
+  printf "%s\n" "$ERROR" "$EXAMPLE";
+  exit 1;
+elif [[ "$echo_port" == "$service_port" ]]; then
+  printf "Ports cannot be the same! Please use different ports.\n";
+  exit 1;
+fi
+
+echo "Starting with echo server on port: $echo_port and the cmb service on port: $service_port"
+
+# Setup -------------------------------------------------------------------------
+
+# clone the cmb service repo
+git clone https://github.com/Unity-Technologies/mps-common-multiplayer-backend.git
+# navigate to the cmb service directory
+cd ./mps-common-multiplayer-backend/runtime
+
+# Install rust
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+# Add the cargo bin directory to the PATH
+export PATH="$HOME/.cargo/bin:$PATH"
+
+
+# Echo server -------------------------------------------------------------------
+
+# Build the echo server
+cargo build --example ngo_echo_server
+# Run the echo server in the background
+cargo run --example ngo_echo_server -- --port $echo_port &
+
+
+# CMB Service -------------------------------------------------------------------
+
+# Build a release version of the standalone cmb service
+cargo build --release --locked
+
+# Run the standalone service on an infinite loop in the background.
+# The infinite loop is required as the service will exit each time all connected clients disconnect.
+# This means the service will exit after each test. The infinite loop will immediately restart the service each time it exits.
+while :; do
+  ./target/release/comb-server -l error --metrics-port 5000 standalone --port $service_port -t 60m;
+done & # <- use & to run the entire loop in the background

com.unity.netcode.gameobjects/Editor/Analytics/NetcodeAnalytics.cs

@@ -29,26 +29,21 @@ internal class NetcodeAnalytics : NetworkManager.NetcodeAnalytics
         /// Determines if we are running an integration test of the analytics integration
         /// </summary>
         internal static bool IsIntegrationTest = false;
+        internal static bool EnableIntegrationTestAnalytics = false;
 #if ENABLE_NGO_ANALYTICS_LOGGING
         internal static bool EnableLogging = true;
 #else
         internal static bool EnableLogging = false;
 #endif
 
-        // Preserves the analytics enabled flag
-        private bool m_OriginalAnalyticsEnabled;
-
         internal override void OnOneTimeSetup()
         {
-            m_OriginalAnalyticsEnabled = EditorAnalytics.enabled;
-            // By default, we always disable analytics during integration testing
-            EditorAnalytics.enabled = false;
+            IsIntegrationTest = true;
         }
 
         internal override void OnOneTimeTearDown()
         {
-            // Reset analytics to the original value
-            EditorAnalytics.enabled = m_OriginalAnalyticsEnabled;
+            IsIntegrationTest = false;
         }
 
         internal List<NetworkManagerAnalyticsHandler> AnalyticsTestResults = new List<NetworkManagerAnalyticsHandler>();
@@ -80,15 +75,19 @@ internal override void ModeChanged(PlayModeStateChange playModeState, NetworkMan
             }
         }
 
+        private bool ShouldLogAnalytics()
+        {
+            return (IsIntegrationTest && EnableIntegrationTestAnalytics) || (!IsIntegrationTest && EditorAnalytics.enabled);
+        }
+
         /// <summary>
         /// Editor Only
         /// Invoked when the session is started.
         /// </summary>
         /// <param name="networkManager">The <see cref="NetworkManager"/> instance when the session is started.</param>
         internal override void SessionStarted(NetworkManager networkManager)
         {
-            // If analytics is disabled and we are not running an integration test, then exit early.
-            if (!EditorAnalytics.enabled && !IsIntegrationTest)
+            if (!ShouldLogAnalytics())
             {
                 return;
             }
@@ -112,7 +111,7 @@ internal override void SessionStarted(NetworkManager networkManager)
         internal override void SessionStopped(NetworkManager networkManager)
         {
             // If analytics is disabled and we are not running an integration test or there are no sessions, then exit early.
-            if ((!EditorAnalytics.enabled && !IsIntegrationTest) || RecentSessions.Count == 0)
+            if (!ShouldLogAnalytics() || RecentSessions.Count == 0)
             {
                 return;
             }
@@ -135,7 +134,7 @@ internal override void SessionStopped(NetworkManager networkManager)
         private void UpdateAnalytics(NetworkManager networkManager)
         {
             // If analytics is disabled and we are not running an integration test or there are no sessions to process, then exit early.
-            if ((!EditorAnalytics.enabled && !IsIntegrationTest) || RecentSessions.Count == 0)
+            if (!ShouldLogAnalytics() || RecentSessions.Count == 0)
             {
                 return;
             }

com.unity.netcode.gameobjects/Runtime/Core/NetworkObject.cs

@@ -628,7 +628,10 @@ public bool SetOwnershipLock(bool lockOwnership = true)
                 RemoveOwnershipExtended(OwnershipStatusExtended.Locked);
             }
 
-            SendOwnershipStatusUpdate();
+            if (IsSpawned)
+            {
+                SendOwnershipStatusUpdate();
+            }
 
             return true;
         }

com.unity.netcode.gameobjects/Runtime/SceneManagement/NetworkSceneManager.cs

@@ -2042,6 +2042,7 @@ internal void SynchronizeNetworkObjects(ulong clientId, bool synchronizingServic
             }
 
             // Organize how (and when) we serialize our NetworkObjects
+            var hasSynchronizedActive = false;
             for (int i = 0; i < SceneManager.sceneCount; i++)
             {
                 var scene = SceneManager.GetSceneAt(i);
@@ -2067,6 +2068,10 @@ internal void SynchronizeNetworkObjects(ulong clientId, bool synchronizingServic
                         continue;
                     }
                     sceneEventData.SceneHash = SceneHashFromNameOrPath(scene.path);
+                    if (sceneEventData.SceneHash == sceneEventData.ActiveSceneHash)
+                    {
+                        hasSynchronizedActive = true;
+                    }
 
                     // If we are just a normal client, then always use the server scene handle
                     if (NetworkManager.DistributedAuthorityMode)
@@ -2085,7 +2090,7 @@ internal void SynchronizeNetworkObjects(ulong clientId, bool synchronizingServic
                 }
 
                 // If we are just a normal client and in distributed authority mode, then always use the known server scene handle
-                if (NetworkManager.DistributedAuthorityMode && !NetworkManager.DAHost)
+                if (NetworkManager.DistributedAuthorityMode && NetworkManager.CMBServiceConnection)
                 {
                     sceneEventData.AddSceneToSynchronize(SceneHashFromNameOrPath(scene.path), ClientSceneHandleToServerSceneHandle[scene.handle]);
                 }
@@ -2095,6 +2100,11 @@ internal void SynchronizeNetworkObjects(ulong clientId, bool synchronizingServic
                 }
             }
 
+            if (!hasSynchronizedActive && NetworkManager.CMBServiceConnection && synchronizingService)
+            {
+                sceneEventData.AddSceneToSynchronize(BuildIndexToHash[activeScene.buildIndex], ClientSceneHandleToServerSceneHandle[activeScene.handle]);
+            }
+
             sceneEventData.AddSpawnedNetworkObjects();
             sceneEventData.AddDespawnedInSceneNetworkObjects();
             var message = new SceneEventMessage

com.unity.netcode.gameobjects/Runtime/Spawning/NetworkSpawnManager.cs

@@ -1118,7 +1118,6 @@ private void SpawnNetworkObjectLocallyCommon(NetworkObject networkObject, ulong
                 return;
             }
 
-            networkObject.IsSpawned = true;
             networkObject.IsSceneObject = sceneObject;
 
             // Always check to make sure our scene of origin is properly set for in-scene placed NetworkObjects
@@ -1151,6 +1150,8 @@ private void SpawnNetworkObjectLocallyCommon(NetworkObject networkObject, ulong
             {
                 networkObject.SetOwnershipLock();
             }
+
+            networkObject.IsSpawned = true;
             SpawnedObjects.Add(networkObject.NetworkObjectId, networkObject);
             SpawnedObjectsList.Add(networkObject);
 
@@ -2228,14 +2229,7 @@ internal void SynchronizeObjectsToNewlyJoinedClient(ulong newClientId)
                 {
                     if (networkObject.Observers.Contains(newClientId))
                     {
-                        if (NetworkManager.LogLevel <= LogLevel.Developer)
-                        {
-                            // Temporary tracking to make sure we are not showing something already visibile (should never be the case for this)
-                            Debug.LogWarning($"[{nameof(SynchronizeObjectsToNewlyJoinedClient)}][{networkObject.name}] New client as already an observer!");
-                        }
-                        // For now, remove the client (impossible for the new client to have an instance since the session owner doesn't) to make sure newly added
-                        // code to handle this edge case works.
-                        networkObject.Observers.Remove(newClientId);
+                        continue;
                     }
                     networkObject.NetworkShow(newClientId);
                 }