There is a need for face capture in the recent project. The following article was referred to at the beginning. To use the officially released Facial AR Remote, we need to build the IOS client by ourselves. Therefore, we need to prepare the development environment including the MacOS operating system and Xcode. After building the Xcode project, I also need to consider the development license and other issues, and when I tried, the Xcode13 version I used had some problems in compilation, which was troublesome.
https://www.163.com/dy/article/E70U8CLT0526E124.html
Another solution was subsequently discovered, namely Live Capture, the IOS client has been released in the App Store under the name Unity Face Capture:
Live Capture is added in Package Manager by git url, address:
Live Capture official manual address:
https://docs.unity.cn/Packages/[email protected]/manual/index.html
PDF document download address:
The documentation is very detailed, you can also refer to the following steps:
1. Create a new empty object and add the Take Recoder component, which depends on the Playable Director component, and the Playable Director is automatically added when the Take Recorder is added:
2. Drag the created model containing the face into the scene, mount the ARKit Face Actor component, and make it a Prefab.
3. Click the + button below the Take Recoder component, add ARKit Face Device, and set its Actor as the face model with the ARKit Face Actor component mounted in step 2:
4. Right-click in the Project window, Create / Live Capture / ARKit Face Capture / Mapper, create a Head Mapper asset, set its Rig Prefab as the Prefab generated in step 2, and set the corresponding Left Eye and RightEye in the face model , Head bone node:
5. Click the Add Renderer button in the Head Mapper asset to bind the BlendShape. If the name is consistent with the requirements in ARKit, it will be automatically bound, otherwise it needs to be set one by one:
6. Assign the edited Head Mapper asset to the Mapper in the ARKit Face Actor component mounted in step 2:
7.Window / Live Capture / Connections Open the Connections window, create a server, and click Start to start:
8. Open the IOS client after startup and click Connect to connect. If the connection cannot be made, check whether the mobile phone and the computer are in the same network segment, and check the computer firewall. It is best to turn them off.
It is also worth noting that the startup of the server is started by clicking Start in the Connections editor window, so its usage environment is in the Unity editor environment. If you want to use it after packaging, you need to remove it from the Package Manager. Migrate to the project Assets directory and create a scripting startup method:
Create a LiveCaptureServer class that inherits Server:
using System;
using System.IO;
using System.Linq;
using System.Text;
using UnityEngine;
using System.Collections.Generic;
using Unity.LiveCapture.Networking;
using Unity.LiveCapture.Networking.Discovery;
namespace Unity.LiveCapture.CompanionApp
{
[CreateAssetMenu(menuName = "Live Capture/Server")]
public class LiveCaptureServer : Server
{
const int k_DefaultPort = 9000;
/// <summary>
/// The server executes this event when a client has connected.
/// </summary>
public static event Action<ICompanionAppClient> ClientConnected = delegate { };
/// <summary>
/// The server executes this event when a client has disconnected.
/// </summary>
public static event Action<ICompanionAppClient> ClientDisconnected = delegate { };
struct ConnectHandler
{
public string Name;
public DateTime Time;
public Func<ICompanionAppClient, bool> Handler;
}
readonly Dictionary<string, Type> s_TypeToClientType = new Dictionary<string, Type>();
static readonly List<ConnectHandler> s_ClientConnectHandlers = new List<ConnectHandler>();
/// <summary>
/// Adds a callback used to take ownership of a client that has connected.
/// </summary>
/// <param name="handler">The callback function. It must return true if it takes ownership of a client.</param>
/// <param name="name">The name of the client to prefer. If set, this handler has priority over clients that have the given name.</param>
/// <param name="time">The time used to determine the priority of handlers when many are listening for the same
/// client <paramref name="name"/>. More recent values have higher priority.</param>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="handler"/> is null.</exception>
public static void RegisterClientConnectHandler(Func<ICompanionAppClient, bool> handler, string name, DateTime time)
{
if (handler == null)
throw new ArgumentNullException(nameof(handler));
DeregisterClientConnectHandler(handler);
s_ClientConnectHandlers.Add(new ConnectHandler
{
Name = name,
Time = time,
Handler = handler,
});
}
/// <summary>
/// Removes a client connection callback.
/// </summary>
/// <param name="handler">The callback to remove.</param>>
/// <exception cref="ArgumentNullException">Thrown if <paramref name="handler"/> is null.</exception>
public static void DeregisterClientConnectHandler(Func<ICompanionAppClient, bool> handler)
{
if (handler == null)
throw new ArgumentNullException(nameof(handler));
for (var i = 0; i < s_ClientConnectHandlers.Count; i++)
{
if (s_ClientConnectHandlers[i].Handler == handler)
{
s_ClientConnectHandlers.RemoveAt(i);
}
}
}
public void Init()
{
foreach (var (type, attributes) in AttributeUtility.GetAllTypes<ClientAttribute>())
{
if (!typeof(CompanionAppClient).IsAssignableFrom(type))
{
Debug.LogError($"{type.FullName} must be assignable from {nameof(CompanionAppClient)} to use the {nameof(ClientAttribute)} attribute.");
continue;
}
foreach (var attribute in attributes)
{
s_TypeToClientType[attribute.Type] = type;
}
}
}
[SerializeField, Tooltip("The TCP port on which the server will listen for incoming connections. Changes to the port only take effect after restarting the server.")]
int m_Port = k_DefaultPort;
[SerializeField, Tooltip("Start the server automatically after entering play mode.")]
bool m_AutoStartOnPlay = true;
readonly DiscoveryServer m_Discovery = new DiscoveryServer();
readonly NetworkServer m_Server = new NetworkServer();
readonly Dictionary<Remote, ICompanionAppClient> m_RemoteToClient = new Dictionary<Remote, ICompanionAppClient>();
/// <summary>
/// The TCP port on which the server will listen for incoming connections.
/// </summary>
/// <remarks>
/// Changes to the port only take effect after restarting the server.
/// </remarks>
public int Port
{
get => m_Port;
set
{
if (m_Port != value)
{
m_Port = value;
OnServerChanged(true);
}
}
}
/// <summary>
/// Start the server automatically after entering play mode.
/// </summary>
public bool AutoStartOnPlay
{
get => m_AutoStartOnPlay;
set
{
if (m_AutoStartOnPlay != value)
{
m_AutoStartOnPlay = value;
OnServerChanged(true);
}
}
}
/// <summary>
/// Are clients able to connect to the server.
/// </summary>
public bool IsRunning => m_Server.IsRunning;
/// <summary>
/// The number of clients currently connected to the server.
/// </summary>
public int ClientCount => m_RemoteToClient.Count;
/// <inheritdoc/>
protected override void OnEnable()
{
base.OnEnable();
m_Server.RemoteConnected += OnClientConnected;
m_Server.RemoteDisconnected += OnClientDisconnected;
}
/// <inheritdoc/>
protected override void OnDisable()
{
base.OnDisable();
m_Discovery.Stop();
m_Server.Stop();
m_Server.RemoteConnected -= OnClientConnected;
m_Server.RemoteDisconnected -= OnClientDisconnected;
}
/// <summary>
/// Gets the currently connected clients.
/// </summary>
/// <returns>A new collection containing the client handles.</returns>
public IEnumerable<ICompanionAppClient> GetClients()
{
return m_RemoteToClient.Values;
}
/// <inheritdoc />
public override string GetName() => "Companion App Server";
/// <summary>
/// Start listening for clients connections.
/// </summary>
public void StartServer()
{
if (!NetworkUtilities.IsPortAvailable(m_Port))
{
Debug.LogError($"Unable to start server: Port {m_Port} is in use by another program! Close the other program, or assign a free port using the Live Capture Window.");
return;
}
if (m_Server.StartServer(m_Port))
{
// start server discovery
var config = new ServerData(
"Live Capture",
Environment.MachineName,
m_Server.ID,
PackageUtility.GetVersion(LiveCaptureInfo.Version)
);
var endPoints = m_Server.EndPoints.ToArray();
m_Discovery.Start(config, endPoints);
}
OnServerChanged(false);
}
/// <summary>
/// Disconnects all clients and stop listening for new connections.
/// </summary>
public void StopServer()
{
m_Server.Stop();
m_Discovery.Stop();
OnServerChanged(false);
}
/// <inheritdoc/>
public override void OnUpdate()
{
m_Server.Update();
m_Discovery.Update();
}
void OnClientConnected(Remote remote)
{
m_Server.RegisterMessageHandler(remote, InitializeClient, false);
}
void OnClientDisconnected(Remote remote, DisconnectStatus status)
{
if (m_RemoteToClient.TryGetValue(remote, out var client))
{
try
{
ClientDisconnected.Invoke(client);
}
catch (Exception e)
{
Debug.LogError(e);
}
m_RemoteToClient.Remove(remote);
OnServerChanged(false);
}
}
void InitializeClient(Message message)
{
try
{
if (message.ChannelType != ChannelType.ReliableOrdered)
{
return;
}
var streamReader = new StreamReader(message.Data, Encoding.UTF8);
var json = streamReader.ReadToEnd();
var data = default(ClientInitialization);
try
{
data = JsonUtility.FromJson<ClientInitialization>(json);
}
catch (Exception)
{
Debug.LogError($"{nameof(CompanionAppServer)} failed to initialize client connection! Could not parse JSON: {json}");
return;
}
if (!s_TypeToClientType.TryGetValue(data.Type, out var clientType))
{
Debug.LogError($"Unknown client type \"{data.Type}\" connected to {nameof(CompanionAppServer)}!");
return;
}
var remote = message.Remote;
var client = Activator.CreateInstance(clientType, m_Server, remote, data) as CompanionAppClient;
client.SendProtocol();
m_RemoteToClient.Add(remote, client);
AssignOwner(client);
ClientConnected.Invoke(client);
OnServerChanged(false);
}
catch (Exception e)
{
Debug.LogException(e);
}
finally
{
message.Dispose();
}
}
void AssignOwner(ICompanionAppClient client)
{
// connect to the registered handler that was most recently used with this client if possible
foreach (var handler in s_ClientConnectHandlers.OrderByDescending(h => h.Time.Ticks))
{
try
{
if (handler.Name == client.Name)
{
if (handler.Handler(client))
return;
}
}
catch (Exception e)
{
Debug.LogException(e);
}
}
// fall back to the first free device that is compatible with the client
foreach (var handler in s_ClientConnectHandlers)
{
try
{
if (handler.Handler(client))
return;
}
catch (Exception e)
{
Debug.LogException(e);
}
}
}
}
}Change the CompanionAppDevice class:
using System;
using UnityEngine;
namespace Unity.LiveCapture.CompanionApp
{
/// <summary>
/// A type of <see cref="LiveCaptureDevice"/> that uses a <see cref="ICompanionAppClient"/> for communication.
/// </summary>
interface ICompanionAppDevice
{
/// <summary>
/// Clears the client assigned to this device.
/// </summary>
void ClearClient();
}
/// <summary>
/// A type of <see cref="LiveCaptureDevice"/> that uses a <see cref="ICompanionAppClient"/> for communication.
/// </summary>
/// <typeparam name="TClient">The type of client this device communicates with.</typeparam>
public abstract class CompanionAppDevice<TClient> : LiveCaptureDevice, ICompanionAppDevice
where TClient : class, ICompanionAppClient
{
bool m_ClientRegistered;
bool m_Recording;
TClient m_Client;
readonly SlateChangeTracker m_SlateChangeTracker = new SlateChangeTracker();
readonly TakeNameFormatter m_TakeNameFormatter = new TakeNameFormatter();
string m_LastAssetName;
bool TryGetInternalClient(out ICompanionAppClientInternal client)
{
client = m_Client as ICompanionAppClientInternal;
return client != null;
}
/// <summary>
/// This function is called when the object becomes enabled and active.
/// </summary>
protected virtual void OnEnable()
{
CompanionAppServer.ClientDisconnected += OnClientDisconnected;
LiveCaptureServer.ClientDisconnected += OnClientDisconnected;
RegisterClient();
}
/// <summary>
/// This function is called when the behaviour becomes disabled.
/// </summary>
/// <remaks>
/// This is also called when the object is destroyed and can be used for any cleanup code.
/// When scripts are reloaded after compilation has finished, OnDisable will be called, followed by an OnEnable after the script has been loaded.
/// </remaks>
protected virtual void OnDisable()
{
CompanionAppServer.ClientDisconnected -= OnClientDisconnected;
CompanionAppServer.DeregisterClientConnectHandler(OnClientConnected);
LiveCaptureServer.ClientConnected -= OnClientDisconnected;
LiveCaptureServer.DeregisterClientConnectHandler(OnClientConnected);
StopRecording();
UnregisterClient();
}
/// <summary>
/// This function is called when the behaviour gets destroyed.
/// </summary>
protected override void OnDestroy()
{
base.OnDestroy();
ClearClient();
}
/// <inheritdoc/>
public override bool IsReady()
{
return m_Client != null;
}
/// <inheritdoc/>
public override bool IsRecording()
{
return m_Recording;
}
/// <inheritdoc/>
public override void StartRecording()
{
if (!m_Recording)
{
m_Recording = true;
OnRecordingChanged();
SendRecordingState();
}
}
/// <inheritdoc/>
public override void StopRecording()
{
if (m_Recording)
{
m_Recording = false;
OnRecordingChanged();
SendRecordingState();
}
}
/// <summary>
/// Gets the client currently assigned to this device.
/// </summary>
/// <returns>The assigned client, or null if none is assigned.</returns>
public TClient GetClient()
{
return m_Client;
}
/// <summary>
/// Assigns a client to this device.
/// </summary>
/// <param name="client">The client to assign, or null to clear the assigned client.</param>
/// <param name="rememberAssignment">Try to auto-assign the client to this device when it reconnects in the future.</param>
public void SetClient(TClient client, bool rememberAssignment)
{
if (m_Client != client)
{
UnregisterClient();
if (m_Client != null)
{
ClientMappingDatabase.DeregisterClientAssociation(this, m_Client, rememberAssignment);
}
m_Client = client;
if (m_Client != null)
{
// if any device is also using this client, we must clear the client from the previous device.
if (ClientMappingDatabase.TryGetDevice(client, out var previousDevice))
{
previousDevice.ClearClient();
}
ClientMappingDatabase.RegisterClientAssociation(this, m_Client, rememberAssignment);
}
RegisterClient();
}
}
void RegisterClient()
{
if (!isActiveAndEnabled || m_ClientRegistered)
{
return;
}
LiveCaptureServer.DeregisterClientConnectHandler(OnClientConnected);
CompanionAppServer.DeregisterClientConnectHandler(OnClientConnected);
m_SlateChangeTracker.Reset();
if (TryGetInternalClient(out var client))
{
client.SetDeviceMode += ClientSetDeviceMode;
client.StartRecording += ClientStartRecording;
client.StopRecording += ClientStopRecording;
client.StartPlayer += ClientStartPlayer;
client.StopPlayer += ClientStopPlayer;
client.PausePlayer += ClientPausePlayer;
client.SetPlayerTime += ClientSetPlayerTime;
client.SetSelectedTake += ClientSetSelectedTake;
client.SetTakeData += ClientSetTakeData;
client.DeleteTake += ClientDeleteTake;
client.SetIterationBase += ClientSetIterationBase;
client.ClearIterationBase += ClientClearIterationBase;
client.TexturePreviewRequested += OnTexturePreviewRequested;
OnClientAssigned();
client.SendInitialize();
UpdateClient();
m_ClientRegistered = true;
}
else
{
ClientMappingDatabase.TryGetClientAssignment(this, out var clientName, out var time);
LiveCaptureServer.RegisterClientConnectHandler(OnClientConnected, clientName, time);
CompanionAppServer.RegisterClientConnectHandler(OnClientConnected, clientName, time);
}
}
void UnregisterClient()
{
if (!m_ClientRegistered)
{
return;
}
if (TryGetInternalClient(out var client))
{
OnClientUnassigned();
client.SendEndSession();
client.SetDeviceMode -= ClientSetDeviceMode;
client.StartRecording -= ClientStartRecording;
client.StopRecording -= ClientStopRecording;
client.StartPlayer -= ClientStartPlayer;
client.StopPlayer -= ClientStopPlayer;
client.PausePlayer -= ClientPausePlayer;
client.SetPlayerTime -= ClientSetPlayerTime;
client.SetSelectedTake -= ClientSetSelectedTake;
client.SetTakeData -= ClientSetTakeData;
client.DeleteTake -= ClientDeleteTake;
client.SetIterationBase -= ClientSetIterationBase;
client.ClearIterationBase -= ClientClearIterationBase;
client.TexturePreviewRequested -= OnTexturePreviewRequested;
m_ClientRegistered = false;
}
}
/// <inheritdoc />
public void ClearClient()
{
SetClient(null, true);
}
/// <summary>
/// Called to send the device state to the client.
/// </summary>
public virtual void UpdateClient()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
SendDeviceState(takeRecorder.IsLive());
var slate = takeRecorder.GetActiveSlate();
var hasSlate = slate != null;
var slateChanged = m_SlateChangeTracker.Changed(slate);
var take = hasSlate ? slate.Take : null;
var assetName = GetAssetName();
var assetNameChanged = assetName != m_LastAssetName;
m_LastAssetName = assetName;
if (TryGetInternalClient(out var client))
{
client.SendFrameRate(takeRecorder.IsLive() || take == null ? takeRecorder.FrameRate : take.FrameRate);
client.SendHasSlate(hasSlate);
client.SendSlateDuration(hasSlate ? slate.Duration : 0d);
client.SendSlateIsPreviewing(takeRecorder.IsPreviewPlaying());
client.SendSlatePreviewTime(takeRecorder.GetPreviewTime());
if (slateChanged || assetNameChanged)
{
if (hasSlate)
m_TakeNameFormatter.ConfigureTake(slate.SceneNumber, slate.ShotName, slate.TakeNumber);
else
m_TakeNameFormatter.ConfigureTake(0, "Shot", 0);
client.SendNextTakeName(m_TakeNameFormatter.GetTakeName());
client.SendNextAssetName(m_TakeNameFormatter.GetAssetName());
}
}
if (slateChanged)
{
SendSlateDescriptor(slate);
}
}
SendRecordingState();
}
/// <summary>
/// Gets the name used for the take asset name.
/// </summary>
/// <returns>The name of the asset.</returns>
protected virtual string GetAssetName() { return name; }
/// <summary>
/// The device calls this method when a new client is assigned.
/// </summary>
protected virtual void OnClientAssigned() {}
/// <summary>
/// The device calls this method when the client is unassigned.
/// </summary>
protected virtual void OnClientUnassigned() {}
/// <summary>
/// The device calls this method when the recording state has changed.
/// </summary>
protected virtual void OnRecordingChanged() {}
/// <summary>
/// The device calls this method when the slate has changed.
/// </summary>
protected virtual void OnSlateChanged(ISlate slate) {}
void ClientStartRecording()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
takeRecorder.StartRecording();
}
Refresh();
}
void ClientStopRecording()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
takeRecorder.StopRecording();
}
Refresh();
}
void ClientSetDeviceMode(DeviceMode deviceMode)
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
takeRecorder.SetLive(deviceMode == DeviceMode.LiveStream);
SendDeviceState(takeRecorder.IsLive());
}
}
void ClientStartPlayer()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
takeRecorder.PlayPreview();
}
Refresh();
}
void ClientStopPlayer()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
takeRecorder.PausePreview();
takeRecorder.SetPreviewTime(0d);
}
Refresh();
}
void ClientPausePlayer()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
takeRecorder.PausePreview();
}
Refresh();
}
void ClientSetPlayerTime(double time)
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
takeRecorder.SetPreviewTime(time);
}
Refresh();
}
void SendDeviceState()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
SendDeviceState(takeRecorder.IsLive());
}
}
void SendDeviceState(bool isLive)
{
if (TryGetInternalClient(out var client))
{
client.SendDeviceMode(isLive ? DeviceMode.LiveStream : DeviceMode.Playback);
}
}
void SendRecordingState()
{
if (TryGetInternalClient(out var client))
{
client.SendRecordingState(IsRecording());
}
}
void SendSlateDescriptor()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
SendSlateDescriptor(takeRecorder.GetActiveSlate());
}
}
void SendSlateDescriptor(ISlate slate)
{
if (TryGetInternalClient(out var client))
{
client.SendSlateDescriptor(SlateDescriptor.Create(slate));
}
OnSlateChanged(slate);
}
void ClientSetSelectedTake(Guid guid)
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
TakeManager.Default.SelectTake(takeRecorder.GetActiveSlate(), guid);
SendSlateDescriptor();
Refresh();
}
}
void ClientSetTakeData(TakeDescriptor descriptor)
{
TakeManager.Default.SetTakeData(descriptor);
SendSlateDescriptor();
Refresh();
}
void ClientDeleteTake(Guid guid)
{
TakeManager.Default.DeleteTake(guid);
SendSlateDescriptor();
Refresh();
}
void ClientSetIterationBase(Guid guid)
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
var slate = takeRecorder.GetActiveSlate();
TakeManager.Default.SetIterationBase(slate, guid);
SendSlateDescriptor(slate);
Refresh();
}
}
void ClientClearIterationBase()
{
var takeRecorder = GetTakeRecorder();
if (takeRecorder != null)
{
var slate = takeRecorder.GetActiveSlate();
TakeManager.Default.ClearIterationBase(slate);
SendSlateDescriptor(slate);
Refresh();
}
}
void OnTexturePreviewRequested(Guid guid)
{
var texture = TakeManager.Default.GetAssetPreview<Texture2D>(guid);
if (texture != null && TryGetInternalClient(out var client))
{
client.SendTexturePreview(guid, texture);
}
}
bool OnClientConnected(ICompanionAppClient client)
{
if (m_Client == null && client is TClient c && (!ClientMappingDatabase.TryGetClientAssignment(this, out var clientName, out _) || c.Name == clientName))
{
SetClient(c, false);
return true;
}
return false;
}
void OnClientDisconnected(ICompanionAppClient client)
{
if (m_Client == client)
{
SetClient(null, false);
}
}
}
}After I packaged it, it ran successfully. The test script:
using UnityEngine;
using Unity.LiveCapture.CompanionApp;
public class LiveCaptureExample : MonoBehaviour
{
LiveCaptureServer server;
private void Awake()
{
server = Resources.Load<LiveCaptureServer>("Live Capture Server");
server.Init();
server.StartServer();
}
private void Update()
{
server.OnUpdate();
}
private void OnDestroy()
{
server.StopServer();
}
}Welcome to the public account "Contemporary Wild Programmer"
