462 lines
17 KiB
C#
462 lines
17 KiB
C#
using NUnit.Framework.Internal;
|
||
using OpenCVForUnity.CoreModule;
|
||
using OpenCVForUnity.DnnModule;
|
||
using OpenCVForUnity.UnityUtils;
|
||
using OpenCVForUnity.UnityUtils.Helper;
|
||
using System;
|
||
using System.Collections;
|
||
using System.Collections.Generic;
|
||
using System.Data;
|
||
using System.Linq;
|
||
using System.Runtime.CompilerServices;
|
||
using System.Xml.Serialization;
|
||
using UnityEngine;
|
||
using UnityEngine.Rendering.Universal;
|
||
using UnityEngine.UI;
|
||
using Yoga;
|
||
|
||
public class YogaManager : MonoSingleton<YogaManager>
|
||
{
|
||
private List<Point> _baseKeyPoints = new List<Point>();
|
||
private Mat _rgbaMat;
|
||
public Mat RgbaMat
|
||
{
|
||
get
|
||
{
|
||
return _rgbaMat;
|
||
}
|
||
|
||
set => _rgbaMat = value;
|
||
}
|
||
|
||
private Queue<(DateTime, List<Point>)> _estimateKeyPointsCache = new Queue<(DateTime, List<Point>)>();
|
||
public Queue<(DateTime, List<Point>)> EstimateKeyPointsCache
|
||
{
|
||
get => _estimateKeyPointsCache;
|
||
}
|
||
|
||
private List<Point> _currEstimateKeyPoints;
|
||
public List<Point> CurrEstimateKeyPoints { get => _currEstimateKeyPoints; set => _currEstimateKeyPoints = value; }
|
||
|
||
public void AddCurrEstimateKeyPoints(List<Point> points)
|
||
{
|
||
_currEstimateKeyPoints = points;
|
||
_estimateKeyPointsCache.Enqueue((DateTime.Now, points));
|
||
if (_estimateKeyPointsCache.Count > YogaConfig.CurrEstimateKeyPointsMaxCount)
|
||
{
|
||
_estimateKeyPointsCache.Dequeue();
|
||
}
|
||
}
|
||
|
||
private List<float[]> _personRectResult = new List<float[]>();
|
||
public List<float[]> PersonRectResult { get => _personRectResult; set => _personRectResult = value; }
|
||
|
||
public List<Point> BaseKeyPoints { get => _baseKeyPoints; set => _baseKeyPoints = value; }
|
||
|
||
private int _currentCheckPointCount; //当前动作计数
|
||
public int CurrentCheckPointCount { get => _currentCheckPointCount; }
|
||
|
||
private int _currentCheckPointSuccessCount; //当前成功动作计数
|
||
public int CurrentCheckPointSuccessCount { get => _currentCheckPointSuccessCount; }
|
||
|
||
public int MaxCheckPointCount => LevelData.MaxCheckPointCount;
|
||
|
||
private int _levelIndex = -1; //用户选择界面选择的动作索引
|
||
public int LevelIndex { get => _levelIndex; internal set => _levelIndex = value; }
|
||
|
||
private Dictionary<AvatarAction, PoseBase> _actions = new Dictionary<AvatarAction, PoseBase>();
|
||
|
||
private bool _isSamplingRunning = false;
|
||
|
||
|
||
public YogaData LevelData
|
||
{
|
||
get
|
||
{
|
||
if (_levelData == null)
|
||
{
|
||
LogPrint.Warning("Action Not Load!", PrintLevel.Normal);
|
||
InitData();
|
||
}
|
||
|
||
return _levelData;
|
||
}
|
||
|
||
set => _levelData = value;
|
||
}
|
||
private Estimator _currentEstimator;
|
||
public Estimator CurrentEstimator { get => _currentEstimator; internal set => _currentEstimator = value; }
|
||
|
||
private YogaData _levelData = null;
|
||
private int _actionIndex = 0;
|
||
private int _comboTimes = 0;
|
||
private bool? _samplingResult = null;
|
||
private bool? _samplingYogaResult;
|
||
|
||
public DateTime ActionStartTime { get; set; }
|
||
public int ActionIndex { get => _actionIndex; internal set => _actionIndex = value; }
|
||
public bool? SamplingResult { get => _samplingResult; }
|
||
|
||
public void InitData()
|
||
{
|
||
_actions[AvatarAction.HeadTurnLeft] = new HeadTurnLeft();
|
||
_actions[AvatarAction.HeadTurnRight] = new HeadTurnRight();
|
||
_actions[AvatarAction.HeadTurnUp] = new HeadTurnUp();
|
||
_actions[AvatarAction.HeadTurnDown] = new HeadTurnDown();
|
||
_actions[AvatarAction.HandsUp] = new HandsUp();
|
||
_actions[AvatarAction.HandsDown] = new HandsHold();
|
||
_actions[AvatarAction.LeftLateralHead] = new LeftLateralHead();
|
||
_actions[AvatarAction.RightLateralHead] = new RightLateralHead();
|
||
_actions[AvatarAction.Hold] = new HoldPosition();
|
||
|
||
_currentCheckPointCount = 0;
|
||
_currentCheckPointSuccessCount = 0;
|
||
_comboTimes = 0;
|
||
|
||
_baseNose = -Vector2.one;
|
||
_baseLEye = -Vector2.one;
|
||
_baseREye = -Vector2.one;
|
||
|
||
//根据用户选择的动作索引，获取相应的资源
|
||
LevelData = YogaDataLoader.LoadData(LevelIndex);
|
||
EventManager.Instance.Dispatch(YogaEventType.UpdateProgress, CurrentCheckPointSuccessCount, CurrentCheckPointCount);
|
||
}
|
||
private void OnEnable()
|
||
{
|
||
EventManager.Instance.AddEventListener(YogaEventType.UI_LevelFinished, ClearingSettlement);
|
||
|
||
EventManager.Instance.AddEventListener(YogaEventType.Action_Success, OnActionSuccess);
|
||
EventManager.Instance.AddEventListener(YogaEventType.Action_Fail, OnActionFailed);
|
||
EventManager.Instance.AddEventListener(YogaEventType.Action_StartSampling, OnSamplingStart);
|
||
EventManager.Instance.AddEventListener(YogaEventType.Action_EndSampling, OnSamplingEnd);
|
||
EventManager.Instance.AddEventListener(YogaEventType.Action_Evaluate, OnEvaluation);
|
||
//extra event
|
||
EventManager.Instance.AddEventListener(YogaEventType.Action_MoveDistanceExactly, ExtraMoveDistanceExactly);
|
||
EventManager.Instance.AddEventListener(YogaEventType.Action_MoveDistanceNotAccurate, ExtraMoveDistanceNotAccurate);
|
||
EventManager.Instance.AddEventListener(YogaEventType.Action_SpeedTooSlow, ExtraSpeedTooSlow);
|
||
EventManager.Instance.AddEventListener(YogaEventType.Action_SpeedTooFast, ExtraSpeedTooFast);
|
||
}
|
||
|
||
private void OnDisable()
|
||
{
|
||
EventManager.Instance.RemoveEventListener(YogaEventType.UI_LevelFinished, ClearingSettlement);
|
||
|
||
EventManager.Instance.RemoveEventListener(YogaEventType.Action_Success, OnActionSuccess);
|
||
EventManager.Instance.RemoveEventListener(YogaEventType.Action_Fail, OnActionFailed);
|
||
EventManager.Instance.RemoveEventListener(YogaEventType.Action_StartSampling, OnSamplingStart);
|
||
EventManager.Instance.RemoveEventListener(YogaEventType.Action_EndSampling, OnSamplingEnd);
|
||
EventManager.Instance.RemoveEventListener(YogaEventType.Action_Evaluate, OnEvaluation);
|
||
|
||
//extra event
|
||
EventManager.Instance.RemoveEventListener(YogaEventType.Action_MoveDistanceExactly, ExtraMoveDistanceExactly);
|
||
EventManager.Instance.AddEventListener(YogaEventType.Action_MoveDistanceNotAccurate, ExtraMoveDistanceNotAccurate);
|
||
EventManager.Instance.RemoveEventListener(YogaEventType.Action_SpeedTooSlow, ExtraSpeedTooSlow);
|
||
EventManager.Instance.RemoveEventListener(YogaEventType.Action_SpeedTooFast, ExtraSpeedTooFast);
|
||
}
|
||
|
||
#region UI相关
|
||
private void OnActionSuccess()
|
||
{
|
||
_currentCheckPointSuccessCount++;
|
||
_currentCheckPointCount++;
|
||
//是否为连续正确动作
|
||
|
||
UpdateData();
|
||
}
|
||
private void OnActionFailed()
|
||
{
|
||
_comboTimes = 0;
|
||
_currentCheckPointCount++;
|
||
UpdateData();
|
||
}
|
||
|
||
private void UpdateData()
|
||
{
|
||
if (CurrentCheckPointCount > MaxCheckPointCount)//如果超过最大数，则停止检测，跳转到下一个动作/下一个动作引导/奖励界面
|
||
{
|
||
LogPrint.Warning("CurrentActionCount > MaxActionCount", PrintLevel.Normal);
|
||
ClearingSettlement();
|
||
}
|
||
|
||
EventManager.Instance.Dispatch(YogaEventType.UpdateProgress, CurrentCheckPointSuccessCount, CurrentCheckPointCount, MaxCheckPointCount);//args[0] = successCount args[1] = excutedCount args[2] = totalCount
|
||
}
|
||
|
||
/// <summary>
|
||
/// 跳转结算界面
|
||
/// </summary>
|
||
public void ClearingSettlement()
|
||
{
|
||
//跳转到奖励界面
|
||
UIManager.Instance.CloseCurrent(); //关闭当前界面并停止检测
|
||
UIManager.Instance.ShowPanel<ClearingSettlementUI>(false, CurrentCheckPointSuccessCount, MaxCheckPointCount);//args[0] = successCount args[1] = totalCount
|
||
_estimateKeyPointsCache.Clear();
|
||
}
|
||
#endregion
|
||
|
||
#region 动作检测相关
|
||
|
||
|
||
//采样质量评估
|
||
public bool? SampleQualityEvaluation(AvatarAction actionType, DateTime startTime, DateTime endTime)
|
||
{
|
||
var pointsSource = new List<(DateTime, List<Point>)>(_estimateKeyPointsCache);
|
||
//获取时间间隔内的所有点及向量变化
|
||
var framePoints = _estimateKeyPointsCache.Where(x => x.Item1 >= startTime && x.Item1 <= endTime).ToList();
|
||
if (framePoints == null)
|
||
{
|
||
LogPrint.Warning("No point has been estimated!");
|
||
framePoints = new List<(DateTime, List<Point>)>();
|
||
}
|
||
|
||
//循环遍历所有的点，检测是否符合最低点数要求
|
||
var checkedPoints = new List<(DateTime, List<Point>)>();
|
||
for (int i = 0; i < framePoints.Count; i++)
|
||
{
|
||
var p = framePoints[i];
|
||
if (!ActionCheckPoints(p.Item2))
|
||
continue;
|
||
checkedPoints.Add(p);
|
||
}
|
||
|
||
if (checkedPoints.Count < 2) //0级实现逻辑
|
||
{
|
||
return NeedMoreData(actionType, startTime, endTime);
|
||
}
|
||
else
|
||
{
|
||
return _actions[actionType].AnalyzingAction(checkedPoints);
|
||
}
|
||
}
|
||
|
||
//需要更多数据
|
||
private bool? NeedMoreData(AvatarAction actionType, DateTime startTime, DateTime endTime)
|
||
{
|
||
//获取1秒后时间间隔内的所有点及向量变化
|
||
var points = _estimateKeyPointsCache.Where(x => x.Item1 >= startTime.AddSeconds(1) && x.Item1 <= endTime.AddSeconds(1)).ToList();
|
||
if (points == null)
|
||
{
|
||
//提示摆正姿势
|
||
LogPrint.Warning("No point has been estimated!");
|
||
return null;
|
||
}
|
||
//检测动作是否正确
|
||
return _actions[actionType].AnalyzingAction(points, true);
|
||
}
|
||
#endregion
|
||
|
||
#region 事件监听
|
||
|
||
private AvatarAction _lastAction = AvatarAction.None;
|
||
private AvatarAction _currentAction = AvatarAction.None;
|
||
|
||
private void OnSamplingStart()
|
||
{
|
||
//获取开始时间标记戳
|
||
ActionStartTime = DateTime.Now;
|
||
_isSamplingRunning = true;
|
||
}
|
||
|
||
private void OnSamplingEnd(params object[] args)
|
||
{
|
||
_isSamplingRunning = false;
|
||
if (args == null || args.Length == 0)
|
||
{
|
||
LogPrint.Error("GetActionBasePoint args is null. Please check animation event trigger configuration.");
|
||
return;
|
||
}
|
||
try
|
||
{
|
||
_lastAction = _currentAction;
|
||
AvatarAction actionType = (AvatarAction)args.FirstOrDefault();
|
||
var startTime = ActionStartTime;
|
||
_samplingResult = SampleQualityEvaluation(actionType, ActionStartTime, DateTime.Now);
|
||
_currentAction = actionType;
|
||
//只有hold动作检测
|
||
if (actionType == AvatarAction.Hold)
|
||
{
|
||
_samplingYogaResult = _samplingResult;
|
||
var pointList = _estimateKeyPointsCache.Where(x => x.Item1 >= startTime && x.Item1 <= DateTime.Now).ToList();
|
||
var nose = getAveragePointVector(pointList, "Nose");
|
||
var neck = getAveragePointVector(pointList, "Neck");
|
||
var LEye = getAveragePointVector(pointList, "LEye");
|
||
var REye = getAveragePointVector(pointList, "REye");
|
||
|
||
if (_lastAction == AvatarAction.HeadTurnLeft)
|
||
{
|
||
if (_baseNose == -Vector2.one)
|
||
return;
|
||
var angle = Mathf.Abs(Vector2.SignedAngle(Vector2.left, (nose - _baseNose)));
|
||
_samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle < 90) && (nose - _baseNose).magnitude > 10;
|
||
}
|
||
else if (_lastAction == AvatarAction.HeadTurnRight)
|
||
{
|
||
if (_baseNose == -Vector2.one)
|
||
return;
|
||
var angle = Mathf.Abs(Vector2.SignedAngle(Vector2.right, (nose - _baseNose)));
|
||
_samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle < 90) && (nose - _baseNose).magnitude > 10;
|
||
}
|
||
else if (_lastAction == AvatarAction.HeadTurnDown)
|
||
{
|
||
if (_baseNose == -Vector2.one)
|
||
return;
|
||
var angle = Mathf.Abs(Vector2.SignedAngle(Vector2.down, -(nose - _baseNose)));//坐标系起始点左上，取y轴位移时需取反
|
||
|
||
LogPrint.Warning($"angle:{angle}, magnitude:{(nose - _baseNose).magnitude}", PrintLevel.Normal);
|
||
_samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle < 90) && (nose - _baseNose).magnitude > 10;
|
||
}
|
||
else if (_lastAction == AvatarAction.HeadTurnUp)
|
||
{
|
||
if (_baseNose == -Vector2.one)
|
||
return;
|
||
var angle = Mathf.Abs(Vector2.SignedAngle(Vector2.up, -(nose - _baseNose))); //坐标系起始点左上，取y轴位移时需取反
|
||
_samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle < 90) && (nose - _baseNose).magnitude > 10;
|
||
}
|
||
else if (_lastAction == AvatarAction.LeftLateralHead)
|
||
{
|
||
if (_baseLEye == -Vector2.one || _baseREye == -Vector2.one)
|
||
return;
|
||
|
||
var angle = Vector2.SignedAngle((LEye - REye), (_baseLEye - _baseREye));
|
||
_samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle > 10);
|
||
}
|
||
else if (_lastAction == AvatarAction.RightLateralHead)
|
||
{
|
||
if (_baseLEye == -Vector2.one || _baseREye == -Vector2.one)
|
||
return;
|
||
var angle = Vector2.SignedAngle((LEye - REye), (_baseLEye - _baseREye));
|
||
_samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle < -10);
|
||
}
|
||
}
|
||
}
|
||
catch (Exception e)
|
||
{
|
||
LogPrint.Exception(e);
|
||
}
|
||
}
|
||
|
||
private void OnEvaluation()
|
||
{
|
||
if (_samplingYogaResult == null || _samplingYogaResult == false)
|
||
{
|
||
_samplingYogaResult = null;
|
||
EventManager.Instance.Dispatch(YogaEventType.Action_Fail);
|
||
return;
|
||
}
|
||
|
||
EventManager.Instance.Dispatch(YogaEventType.Action_Success);
|
||
_comboTimes++;
|
||
if (_comboTimes >= 3)
|
||
{
|
||
AudioManager.Instance.PlayCVInQueue("WellDone");
|
||
}
|
||
else if (_comboTimes == 2)
|
||
{
|
||
AudioManager.Instance.PlayCVInQueue("Nice");
|
||
}
|
||
else
|
||
{
|
||
AudioManager.Instance.PlayCVInQueue("Good");
|
||
}
|
||
_samplingYogaResult = null;
|
||
}
|
||
/// extra
|
||
private void ExtraMoveDistanceExactly()
|
||
{
|
||
AudioManager.Instance.PlayCVInQueue("ExactlyCorrect");
|
||
}
|
||
private void ExtraMoveDistanceNotAccurate()
|
||
{
|
||
AudioManager.Instance.PlayCVInQueue("NotAccurate");
|
||
}
|
||
|
||
private void ExtraSpeedTooFast()
|
||
{
|
||
AudioManager.Instance.PlayCVInQueue("SpeedTooFast");
|
||
}
|
||
|
||
private void ExtraSpeedTooSlow()
|
||
{
|
||
AudioManager.Instance.PlayCVInQueue("SpeedTooSlow");
|
||
}
|
||
#endregion
|
||
|
||
|
||
public bool ActionCheckPoints(List<Point> points)
|
||
{
|
||
if (points == null || points.Count == 0)
|
||
{
|
||
LogPrint.Log("ActionCheckPoints points is null");
|
||
return false;
|
||
}
|
||
|
||
//for (int i = 0; i < points.Count; i++)
|
||
//{
|
||
// Point p = points[i];
|
||
// if (p != new Point(-1, -1))
|
||
// LogPrint.Warning($"ActionPoints p({i}): {i.tagName()}, value: {p.x},{p.y}");
|
||
//}
|
||
|
||
foreach (var p in LevelData.MustPoints)
|
||
{
|
||
if (!p.IsValid(points)) //有一个点不符合 返回false
|
||
{
|
||
//LogPrint.Log($"ActionCheckPoints failed, {p} is not valid");
|
||
return false;
|
||
}
|
||
}
|
||
|
||
//不设的情况下，不检测
|
||
if (LevelData.AnyPoints != null && LevelData.AnyPoints.Count > 0)
|
||
{
|
||
foreach (var p in LevelData.AnyPoints)
|
||
{
|
||
if (p.IsValid(points)) //有一个点符合 返回true
|
||
{
|
||
return true;
|
||
}
|
||
}
|
||
return false; //没有一个点符合 返回false
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
private Vector2 _baseNose = -Vector2.one;
|
||
private Vector2 _baseNeck = -Vector2.one;
|
||
private Vector2 _baseLEye = -Vector2.one;
|
||
private Vector2 _baseREye = -Vector2.one;
|
||
|
||
public void SetBasePosPoint()
|
||
{
|
||
//获取近1秒内所有关键点的平均值
|
||
var cacheCopy = new List<(DateTime, List<Point>)>(_estimateKeyPointsCache);
|
||
var poinsList = cacheCopy.Where(x => x.Item1 >= DateTime.Now.AddSeconds(-1) && x.Item1 <= DateTime.Now).ToList();
|
||
|
||
_baseNose = getAveragePointVector(poinsList, "Nose");
|
||
_baseNeck = getAveragePointVector(poinsList, "Neck");
|
||
_baseLEye = getAveragePointVector(poinsList, "LEye");
|
||
_baseREye = getAveragePointVector(poinsList, "REye");
|
||
}
|
||
|
||
private Vector2 getAveragePointVector(List<(DateTime, List<Point>)> poinsList, string partName)
|
||
{
|
||
int index = 0;
|
||
Vector2 retVal = -Vector2.one;
|
||
Vector2 totalVector = Vector2.zero;
|
||
foreach (var item in poinsList)
|
||
{
|
||
Vector2 vector = partName.vector(item.Item2);
|
||
if (vector != -Vector2.one)
|
||
{
|
||
totalVector += vector;
|
||
index++;
|
||
}
|
||
}
|
||
if (index != 0)
|
||
{
|
||
retVal = totalVector / index;
|
||
}
|
||
|
||
return retVal;
|
||
}
|
||
} |