385 lines
13 KiB
C#
385 lines
13 KiB
C#
using NUnit.Framework.Internal;
|
|
using OpenCVForUnity.CoreModule;
|
|
using OpenCVForUnity.DnnModule;
|
|
using OpenCVForUnity.UnityUtils;
|
|
using OpenCVForUnity.UnityUtils.Helper;
|
|
using System;
|
|
using System.Collections;
|
|
using System.Collections.Generic;
|
|
using System.Data;
|
|
using System.Linq;
|
|
using System.Runtime.CompilerServices;
|
|
using UnityEngine;
|
|
using UnityEngine.UI;
|
|
using Yoga;
|
|
|
|
public class YogaManager : MonoSingleton<YogaManager>
|
|
{
|
|
private List<Point> _baseKeyPoints = new List<Point>();
|
|
private Mat _rgbaMat;
|
|
public Mat RgbaMat
|
|
{
|
|
get
|
|
{
|
|
return _rgbaMat;
|
|
}
|
|
|
|
set => _rgbaMat = value;
|
|
}
|
|
|
|
protected bool _isCorrectAction = false;
|
|
private Queue<(DateTime, List<Point>)> _estimateKeyPointsCache = new Queue<(DateTime, List<Point>)>();
|
|
public Queue<(DateTime, List<Point>)> EstimateKeyPointsCache
|
|
{
|
|
get => _estimateKeyPointsCache;
|
|
}
|
|
|
|
private List<Point> _currEstimateKeyPoints;
|
|
public List<Point> CurrEstimateKeyPoints { get => _currEstimateKeyPoints; set => _currEstimateKeyPoints = value; }
|
|
|
|
public void AddCurrEstimateKeyPoints(List<Point> points)
|
|
{
|
|
_currEstimateKeyPoints = points;
|
|
_estimateKeyPointsCache.Enqueue((DateTime.Now, points));
|
|
if (_estimateKeyPointsCache.Count > YogaConfig.CurrEstimateKeyPointsMaxCount)
|
|
{
|
|
_estimateKeyPointsCache.Dequeue();
|
|
}
|
|
}
|
|
|
|
private List<float[]> _personRectResult = new List<float[]>();
|
|
public List<float[]> PersonRectResult { get => _personRectResult; set => _personRectResult = value; }
|
|
|
|
public List<Point> BaseKeyPoints { get => _baseKeyPoints; set => _baseKeyPoints = value; }
|
|
|
|
private int _currentActionCount; //当前动作计数
|
|
public int CurrentActionCount { get => _currentActionCount; }
|
|
|
|
private int _currentSuccessActionCount; //当前成功动作计数
|
|
public int CurrentSuccessActionCount { get => _currentSuccessActionCount; }
|
|
|
|
public int MaxActionCount => LevelData.MaxActionCount;
|
|
|
|
private int _actionIndex = -1; //用户选择界面选择的动作索引
|
|
public int ActionIndex { get => _actionIndex; internal set => _actionIndex = value; }
|
|
|
|
private Dictionary<AvatarAction, PoseBase> _actions = new Dictionary<AvatarAction, PoseBase>();
|
|
|
|
private bool _isActionRunning = false;
|
|
|
|
|
|
public YogaData LevelData
|
|
{
|
|
get
|
|
{
|
|
if (_levelData == null)
|
|
{
|
|
LogPrint.Warning("Action Not Load!", PrintLevel.Normal);
|
|
InitData();
|
|
}
|
|
|
|
return _levelData;
|
|
}
|
|
|
|
set => _levelData = value;
|
|
}
|
|
private Estimator _currentEstimator;
|
|
public Estimator CurrentEstimator { get => _currentEstimator; internal set => _currentEstimator = value; }
|
|
|
|
private YogaData _levelData = null;
|
|
public DateTime ActionStartTime { get; set; }
|
|
public void InitData()
|
|
{
|
|
_actions[AvatarAction.HeadTurnLeft] = new HeadTurnLeft();
|
|
_actions[AvatarAction.HeadTurnRight] = new HeadTurnRight();
|
|
_actions[AvatarAction.HeadTurnUp] = new HeadTurnUp();
|
|
_actions[AvatarAction.HeadTurnDown] = new HeadTurnDown();
|
|
_actions[AvatarAction.HandsUp] = new HandsUp();
|
|
_actions[AvatarAction.HandsDown] = new HandsHold();
|
|
|
|
_currentActionCount = 0;
|
|
_currentSuccessActionCount = 0;
|
|
|
|
//根据用户选择的动作索引，获取相应的资源
|
|
LevelData = YogaDataLoader.LoadData(ActionIndex);
|
|
EventManager.Instance.Dispatch(YogaEventType.UpdateProgress, CurrentSuccessActionCount, CurrentActionCount);
|
|
}
|
|
private void OnEnable()
|
|
{
|
|
EventManager.Instance.AddEventListener(YogaEventType.UI_ScoreUpdate, ScoreUpdate);
|
|
EventManager.Instance.AddEventListener(YogaEventType.UI_LevelFinished, ClearingSettlement);
|
|
|
|
EventManager.Instance.AddEventListener(YogaEventType.Action_Success, OnActionSuccess);
|
|
EventManager.Instance.AddEventListener(YogaEventType.Action_Fail, OnActionFailed);
|
|
EventManager.Instance.AddEventListener(YogaEventType.Action_Start, OnActionStart);
|
|
EventManager.Instance.AddEventListener(YogaEventType.Action_End, OnActionEnd);
|
|
//extra event
|
|
EventManager.Instance.AddEventListener(YogaEventType.Action_MoveDistanceExactly, ExtraMoveDistanceExactly);
|
|
EventManager.Instance.AddEventListener(YogaEventType.Action_MoveDistanceNotAccurate, ExtraMoveDistanceNotAccurate);
|
|
EventManager.Instance.AddEventListener(YogaEventType.Action_SpeedTooSlow, ExtraSpeedTooSlow);
|
|
EventManager.Instance.AddEventListener(YogaEventType.Action_SpeedTooFast, ExtraSpeedTooFast);
|
|
}
|
|
|
|
|
|
private void OnDisable()
|
|
{
|
|
EventManager.Instance.RemoveEventListener(YogaEventType.UI_ScoreUpdate, ScoreUpdate);
|
|
EventManager.Instance.RemoveEventListener(YogaEventType.UI_LevelFinished, ClearingSettlement);
|
|
|
|
EventManager.Instance.RemoveEventListener(YogaEventType.Action_Success, OnActionSuccess);
|
|
EventManager.Instance.RemoveEventListener(YogaEventType.Action_Fail, OnActionFailed);
|
|
EventManager.Instance.RemoveEventListener(YogaEventType.Action_Start, OnActionStart);
|
|
EventManager.Instance.RemoveEventListener(YogaEventType.Action_End, OnActionEnd);
|
|
//extra event
|
|
EventManager.Instance.RemoveEventListener(YogaEventType.Action_MoveDistanceExactly, ExtraMoveDistanceExactly);
|
|
EventManager.Instance.AddEventListener(YogaEventType.Action_MoveDistanceNotAccurate, ExtraMoveDistanceNotAccurate);
|
|
EventManager.Instance.RemoveEventListener(YogaEventType.Action_SpeedTooSlow, ExtraSpeedTooSlow);
|
|
EventManager.Instance.RemoveEventListener(YogaEventType.Action_SpeedTooFast, ExtraSpeedTooFast);
|
|
}
|
|
|
|
#region UI相关
|
|
private void OnActionSuccess()
|
|
{
|
|
_currentSuccessActionCount++;
|
|
_currentActionCount++;
|
|
AudioManager.Instance.PlayCVInQueue("Correct");
|
|
UpdateData();
|
|
}
|
|
private void OnActionFailed()
|
|
{
|
|
_currentActionCount++;
|
|
AudioManager.Instance.PlayCVInQueue("Wrong");
|
|
UpdateData();
|
|
}
|
|
|
|
private void UpdateData()
|
|
{
|
|
if (CurrentActionCount > MaxActionCount)//如果超过最大数，则停止检测，跳转到下一个动作/下一个动作引导/奖励界面
|
|
{
|
|
LogPrint.Warning("CurrentActionCount > MaxActionCount", PrintLevel.Normal);
|
|
ClearingSettlement();
|
|
}
|
|
|
|
EventManager.Instance.Dispatch(YogaEventType.UpdateProgress, CurrentSuccessActionCount, CurrentActionCount, MaxActionCount);//args[0] = successCount args[1] = excutedCount args[2] = totalCount
|
|
}
|
|
|
|
/// <summary>
|
|
/// 跳转结算界面
|
|
/// </summary>
|
|
public void ClearingSettlement()
|
|
{
|
|
//跳转到奖励界面
|
|
UIManager.Instance.CloseCurrent(); //关闭当前界面并停止检测
|
|
UIManager.Instance.ShowPanel<ClearingSettlementUI>(false, CurrentSuccessActionCount, MaxActionCount);//args[0] = successCount args[1] = totalCount
|
|
}
|
|
|
|
public void ScoreUpdate()
|
|
{
|
|
|
|
if (_isCorrectAction)
|
|
{
|
|
EventManager.Instance.Dispatch(YogaEventType.Action_Success);
|
|
}
|
|
else
|
|
{
|
|
EventManager.Instance.Dispatch(YogaEventType.Action_Fail);
|
|
}
|
|
|
|
_isCorrectAction = false;//重置
|
|
}
|
|
#endregion
|
|
|
|
#region 动作检测相关
|
|
|
|
|
|
//采样质量评估
|
|
public void SampleQualityEvaluation(AvatarAction actionType, DateTime startTime, DateTime endTime)
|
|
{
|
|
//获取时间间隔内的所有点及向量变化
|
|
var framePoints = _estimateKeyPointsCache.Where(x => x.Item1 >= startTime && x.Item1 <= endTime).ToList();
|
|
if (framePoints == null)
|
|
{
|
|
LogPrint.Warning("No point has been estimated!");
|
|
framePoints = new List<(DateTime, List<Point>)>();
|
|
}
|
|
|
|
//循环遍历所有的点，检测是否符合最低点数要求
|
|
var checkedPoints = new List<(DateTime, List<Point>)>();
|
|
for (int i = 0; i < framePoints.Count; i++)
|
|
{
|
|
var p = framePoints[i];
|
|
if (!ActionCheckPoints(p.Item2))
|
|
continue;
|
|
checkedPoints.Add(p);
|
|
}
|
|
|
|
if (checkedPoints.Count < 2) //0级实现逻辑
|
|
{
|
|
NeedMoreData(actionType, startTime, endTime);
|
|
}
|
|
else
|
|
{
|
|
_actions[actionType].AnalyzingAction(checkedPoints);
|
|
}
|
|
}
|
|
|
|
//需要更多数据
|
|
private void NeedMoreData(AvatarAction actionType, DateTime startTime, DateTime endTime)
|
|
{
|
|
//获取1秒后时间间隔内的所有点及向量变化
|
|
var points = _estimateKeyPointsCache.Where(x => x.Item1 >= startTime.AddSeconds(1) && x.Item1 <= endTime.AddSeconds(1)).ToList();
|
|
if (points == null)
|
|
{
|
|
//提示摆正姿势
|
|
LogPrint.Warning("No point has been estimated!");
|
|
return;
|
|
}
|
|
//检测动作是否正确
|
|
_actions[actionType].AnalyzingAction(points, true);
|
|
}
|
|
#endregion
|
|
|
|
#region 事件监听
|
|
|
|
private void OnActionStart(params object[] args)
|
|
{
|
|
//如果上一次动作还未结束
|
|
if (_isActionRunning)
|
|
{
|
|
//结束上一次动作
|
|
OnActionEnd();
|
|
}
|
|
|
|
if (args == null || args.Length < 2)
|
|
{
|
|
LogPrint.Error("GetActionBasePoint args is null. Please check animation event trigger configuration.");
|
|
return;
|
|
}
|
|
try
|
|
{
|
|
AvatarAction actionType = (AvatarAction)args.FirstOrDefault();
|
|
bool isMoveBack = (bool)args.LastOrDefault();
|
|
LogPrint.Log($"PlayGuideVoice: {actionType}", PrintLevel.Normal);
|
|
if (isMoveBack)
|
|
{
|
|
AudioManager.Instance.PlayCVInQueue(actionType.ToString());
|
|
}
|
|
else
|
|
{
|
|
AudioManager.Instance.PlayCVInQueue("BackToPosition");
|
|
}
|
|
|
|
//获取开始时间标记戳
|
|
ActionStartTime = DateTime.Now;
|
|
_isActionRunning = true;
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
LogPrint.Exception(e);
|
|
return;
|
|
}
|
|
}
|
|
|
|
private void OnActionEnd(params object[] args)
|
|
{
|
|
//如果上一次动作已经结束
|
|
if (!_isActionRunning)
|
|
return;
|
|
|
|
if (args == null || args.Length == 0)
|
|
{
|
|
LogPrint.Error("GetActionBasePoint args is null. Please check animation event trigger configuration.");
|
|
return;
|
|
}
|
|
|
|
try
|
|
{
|
|
AvatarAction actionType = (AvatarAction)args.FirstOrDefault();
|
|
bool isActionEnd = (bool)args.LastOrDefault();
|
|
var startTime = ActionStartTime;
|
|
var endTime = DateTime.Now;
|
|
|
|
LogPrint.Log($"PlayGuideVoice: {actionType}", PrintLevel.Normal);
|
|
if (isActionEnd)
|
|
{
|
|
AudioManager.Instance.PlayCVInQueue("End");
|
|
}
|
|
else
|
|
{
|
|
AudioManager.Instance.PlayCVInQueue("Hold");
|
|
}
|
|
|
|
//结束上一次动作
|
|
//对动作进行采样质量评估
|
|
SampleQualityEvaluation(actionType, startTime, endTime);
|
|
_isActionRunning = false;
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
LogPrint.Exception(e);
|
|
return;
|
|
}
|
|
}
|
|
|
|
/// extra
|
|
private void ExtraMoveDistanceExactly()
|
|
{
|
|
AudioManager.Instance.PlayCVInQueue("ExactlyCorrect");
|
|
}
|
|
private void ExtraMoveDistanceNotAccurate()
|
|
{
|
|
AudioManager.Instance.PlayCVInQueue("NotAccurate");
|
|
}
|
|
|
|
private void ExtraSpeedTooFast()
|
|
{
|
|
AudioManager.Instance.PlayCVInQueue("SpeedTooFast");
|
|
}
|
|
|
|
private void ExtraSpeedTooSlow()
|
|
{
|
|
AudioManager.Instance.PlayCVInQueue("SpeedTooSlow");
|
|
}
|
|
#endregion
|
|
|
|
|
|
public bool ActionCheckPoints(List<Point> points)
|
|
{
|
|
if (points == null || points.Count == 0)
|
|
{
|
|
LogPrint.Log("ActionCheckPoints points is null");
|
|
return false;
|
|
}
|
|
|
|
//for (int i = 0; i < points.Count; i++)
|
|
//{
|
|
// Point p = points[i];
|
|
// if (p != new Point(-1, -1))
|
|
// LogPrint.Warning($"ActionPoints p({i}): {i.tagName()}, value: {p.x},{p.y}");
|
|
//}
|
|
|
|
foreach (var p in LevelData.MustPoints)
|
|
{
|
|
if (!p.IsValid(points)) //有一个点不符合 返回false
|
|
{
|
|
//LogPrint.Log($"ActionCheckPoints failed, {p} is not valid");
|
|
return false;
|
|
}
|
|
}
|
|
|
|
//不设的情况下，不检测
|
|
if (LevelData.AnyPoints != null && LevelData.AnyPoints.Count > 0)
|
|
{
|
|
foreach (var p in LevelData.AnyPoints)
|
|
{
|
|
if (p.IsValid(points)) //有一个点符合 返回true
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
return false; //没有一个点符合 返回false
|
|
}
|
|
|
|
return true;
|
|
}
|
|
} |