using NUnit.Framework.Internal; using OpenCVForUnity.CoreModule; using OpenCVForUnity.DnnModule; using OpenCVForUnity.UnityUtils; using OpenCVForUnity.UnityUtils.Helper; using System; using System.Collections; using System.Collections.Generic; using System.Data; using System.Linq; using System.Runtime.CompilerServices; using System.Xml.Serialization; using UnityEngine; using UnityEngine.Rendering.Universal; using UnityEngine.UI; using Yoga; public class YogaManager : MonoSingleton { private List _baseKeyPoints = new List(); private Mat _rgbaMat; public Mat RgbaMat { get { return _rgbaMat; } set => _rgbaMat = value; } private Queue<(DateTime, List)> _estimateKeyPointsCache = new Queue<(DateTime, List)>(); public Queue<(DateTime, List)> EstimateKeyPointsCache { get => _estimateKeyPointsCache; } private List _currEstimateKeyPoints; public List CurrEstimateKeyPoints { get => _currEstimateKeyPoints; set => _currEstimateKeyPoints = value; } public void AddCurrEstimateKeyPoints(List points) { _currEstimateKeyPoints = points; _estimateKeyPointsCache.Enqueue((DateTime.Now, points)); if (_estimateKeyPointsCache.Count > YogaConfig.CurrEstimateKeyPointsMaxCount) { _estimateKeyPointsCache.Dequeue(); } } private List _personRectResult = new List(); public List PersonRectResult { get => _personRectResult; set => _personRectResult = value; } public List BaseKeyPoints { get => _baseKeyPoints; set => _baseKeyPoints = value; } private int _currentCheckPointCount; //当前动作计数 public int CurrentCheckPointCount { get => _currentCheckPointCount; } private int _currentCheckPointSuccessCount; //当前成功动作计数 public int CurrentCheckPointSuccessCount { get => _currentCheckPointSuccessCount; } public int MaxCheckPointCount => LevelData.MaxCheckPointCount; private int _levelIndex = -1; //用户选择界面选择的动作索引 public int LevelIndex { get => _levelIndex; internal set => _levelIndex = value; } private Dictionary _actions = new Dictionary(); private bool _isSamplingRunning = false; public YogaData LevelData { get { if (_levelData == null) { LogPrint.Warning("Action Not Load!", PrintLevel.Normal); InitData(); } return _levelData; } set => _levelData = value; } private Estimator _currentEstimator; public Estimator CurrentEstimator { get => _currentEstimator; internal set => _currentEstimator = value; } private YogaData _levelData = null; private int _actionIndex = 0; private int _comboTimes = 0; private bool? _samplingResult = null; private bool? _samplingYogaResult; public DateTime ActionStartTime { get; set; } public int ActionIndex { get => _actionIndex; internal set => _actionIndex = value; } public bool? SamplingResult { get => _samplingResult; } public void InitData() { _actions[AvatarAction.HeadTurnLeft] = new HeadTurnLeft(); _actions[AvatarAction.HeadTurnRight] = new HeadTurnRight(); _actions[AvatarAction.HeadTurnUp] = new HeadTurnUp(); _actions[AvatarAction.HeadTurnDown] = new HeadTurnDown(); _actions[AvatarAction.HandsUp] = new HandsUp(); _actions[AvatarAction.HandsDown] = new HandsHold(); _actions[AvatarAction.LeftLateralHead] = new LeftLateralHead(); _actions[AvatarAction.RightLateralHead] = new RightLateralHead(); _actions[AvatarAction.Hold] = new HoldPosition(); _currentCheckPointCount = 0; _currentCheckPointSuccessCount = 0; _comboTimes = 0; _baseNose = -Vector2.one; _baseLEye = -Vector2.one; _baseREye = -Vector2.one; //根据用户选择的动作索引,获取相应的资源 LevelData = YogaDataLoader.LoadData(LevelIndex); EventManager.Instance.Dispatch(YogaEventType.UpdateProgress, CurrentCheckPointSuccessCount, CurrentCheckPointCount); } private void OnEnable() { EventManager.Instance.AddEventListener(YogaEventType.UI_LevelFinished, ClearingSettlement); EventManager.Instance.AddEventListener(YogaEventType.Action_Success, OnActionSuccess); EventManager.Instance.AddEventListener(YogaEventType.Action_Fail, OnActionFailed); EventManager.Instance.AddEventListener(YogaEventType.Action_StartSampling, OnSamplingStart); EventManager.Instance.AddEventListener(YogaEventType.Action_EndSampling, OnSamplingEnd); EventManager.Instance.AddEventListener(YogaEventType.Action_Evaluate, OnEvaluation); //extra event EventManager.Instance.AddEventListener(YogaEventType.Action_MoveDistanceExactly, ExtraMoveDistanceExactly); EventManager.Instance.AddEventListener(YogaEventType.Action_MoveDistanceNotAccurate, ExtraMoveDistanceNotAccurate); EventManager.Instance.AddEventListener(YogaEventType.Action_SpeedTooSlow, ExtraSpeedTooSlow); EventManager.Instance.AddEventListener(YogaEventType.Action_SpeedTooFast, ExtraSpeedTooFast); } private void OnDisable() { EventManager.Instance.RemoveEventListener(YogaEventType.UI_LevelFinished, ClearingSettlement); EventManager.Instance.RemoveEventListener(YogaEventType.Action_Success, OnActionSuccess); EventManager.Instance.RemoveEventListener(YogaEventType.Action_Fail, OnActionFailed); EventManager.Instance.RemoveEventListener(YogaEventType.Action_StartSampling, OnSamplingStart); EventManager.Instance.RemoveEventListener(YogaEventType.Action_EndSampling, OnSamplingEnd); EventManager.Instance.RemoveEventListener(YogaEventType.Action_Evaluate, OnEvaluation); //extra event EventManager.Instance.RemoveEventListener(YogaEventType.Action_MoveDistanceExactly, ExtraMoveDistanceExactly); EventManager.Instance.AddEventListener(YogaEventType.Action_MoveDistanceNotAccurate, ExtraMoveDistanceNotAccurate); EventManager.Instance.RemoveEventListener(YogaEventType.Action_SpeedTooSlow, ExtraSpeedTooSlow); EventManager.Instance.RemoveEventListener(YogaEventType.Action_SpeedTooFast, ExtraSpeedTooFast); } #region UI相关 private void OnActionSuccess() { _currentCheckPointSuccessCount++; _currentCheckPointCount++; //是否为连续正确动作 UpdateData(); } private void OnActionFailed() { _comboTimes = 0; _currentCheckPointCount++; UpdateData(); } private void UpdateData() { if (CurrentCheckPointCount > MaxCheckPointCount)//如果超过最大数,则停止检测,跳转到下一个动作/下一个动作引导/奖励界面 { LogPrint.Warning("CurrentActionCount > MaxActionCount", PrintLevel.Normal); ClearingSettlement(); } EventManager.Instance.Dispatch(YogaEventType.UpdateProgress, CurrentCheckPointSuccessCount, CurrentCheckPointCount, MaxCheckPointCount);//args[0] = successCount args[1] = excutedCount args[2] = totalCount } /// /// 跳转结算界面 /// public void ClearingSettlement() { //跳转到奖励界面 UIManager.Instance.CloseCurrent(); //关闭当前界面并停止检测 UIManager.Instance.ShowPanel(false, CurrentCheckPointSuccessCount, MaxCheckPointCount);//args[0] = successCount args[1] = totalCount _estimateKeyPointsCache.Clear(); } #endregion #region 动作检测相关 //采样质量评估 public bool? SampleQualityEvaluation(AvatarAction actionType, DateTime startTime, DateTime endTime) { var pointsSource = new List<(DateTime, List)>(_estimateKeyPointsCache); //获取时间间隔内的所有点及向量变化 var framePoints = _estimateKeyPointsCache.Where(x => x.Item1 >= startTime && x.Item1 <= endTime).ToList(); if (framePoints == null) { LogPrint.Warning("No point has been estimated!"); framePoints = new List<(DateTime, List)>(); } //循环遍历所有的点,检测是否符合最低点数要求 var checkedPoints = new List<(DateTime, List)>(); for (int i = 0; i < framePoints.Count; i++) { var p = framePoints[i]; if (!ActionCheckPoints(p.Item2)) continue; checkedPoints.Add(p); } if (checkedPoints.Count < 2) //0级实现逻辑 { return NeedMoreData(actionType, startTime, endTime); } else { return _actions[actionType].AnalyzingAction(checkedPoints); } } //需要更多数据 private bool? NeedMoreData(AvatarAction actionType, DateTime startTime, DateTime endTime) { //获取1秒后时间间隔内的所有点及向量变化 var points = _estimateKeyPointsCache.Where(x => x.Item1 >= startTime.AddSeconds(1) && x.Item1 <= endTime.AddSeconds(1)).ToList(); if (points == null) { //提示摆正姿势 LogPrint.Warning("No point has been estimated!"); return null; } //检测动作是否正确 return _actions[actionType].AnalyzingAction(points, true); } #endregion #region 事件监听 private AvatarAction _lastAction = AvatarAction.None; private AvatarAction _currentAction = AvatarAction.None; private void OnSamplingStart() { //获取开始时间标记戳 ActionStartTime = DateTime.Now; _isSamplingRunning = true; } private void OnSamplingEnd(params object[] args) { _isSamplingRunning = false; if (args == null || args.Length == 0) { LogPrint.Error("GetActionBasePoint args is null. Please check animation event trigger configuration."); return; } try { _lastAction = _currentAction; AvatarAction actionType = (AvatarAction)args.FirstOrDefault(); var startTime = ActionStartTime; _samplingResult = SampleQualityEvaluation(actionType, ActionStartTime, DateTime.Now); _currentAction = actionType; //只有hold动作检测 if (actionType == AvatarAction.Hold) { _samplingYogaResult = _samplingResult; var pointList = _estimateKeyPointsCache.Where(x => x.Item1 >= startTime && x.Item1 <= DateTime.Now).ToList(); var nose = getAveragePointVector(pointList, "Nose"); var neck = getAveragePointVector(pointList, "Neck"); var LEye = getAveragePointVector(pointList, "LEye"); var REye = getAveragePointVector(pointList, "REye"); if (_lastAction == AvatarAction.HeadTurnLeft) { if (_baseNose == -Vector2.one) return; var angle = Mathf.Abs(Vector2.SignedAngle(Vector2.left, (nose - _baseNose))); _samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle < 90) && (nose - _baseNose).magnitude > 10; } else if (_lastAction == AvatarAction.HeadTurnRight) { if (_baseNose == -Vector2.one) return; var angle = Mathf.Abs(Vector2.SignedAngle(Vector2.right, (nose - _baseNose))); _samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle < 90) && (nose - _baseNose).magnitude > 10; } else if (_lastAction == AvatarAction.HeadTurnDown) { if (_baseNose == -Vector2.one) return; var angle = Mathf.Abs(Vector2.SignedAngle(Vector2.down, -(nose - _baseNose)));//坐标系起始点左上,取y轴位移时需取反 LogPrint.Warning($"angle:{angle}, magnitude:{(nose - _baseNose).magnitude}", PrintLevel.Normal); _samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle < 90) && (nose - _baseNose).magnitude > 10; } else if (_lastAction == AvatarAction.HeadTurnUp) { if (_baseNose == -Vector2.one) return; var angle = Mathf.Abs(Vector2.SignedAngle(Vector2.up, -(nose - _baseNose))); //坐标系起始点左上,取y轴位移时需取反 _samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle < 90) && (nose - _baseNose).magnitude > 10; } else if (_lastAction == AvatarAction.LeftLateralHead) { if (_baseLEye == -Vector2.one || _baseREye == -Vector2.one) return; var angle = Vector2.SignedAngle((LEye - REye), (_baseLEye - _baseREye)); _samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle > 10); } else if (_lastAction == AvatarAction.RightLateralHead) { if (_baseLEye == -Vector2.one || _baseREye == -Vector2.one) return; var angle = Vector2.SignedAngle((LEye - REye), (_baseLEye - _baseREye)); _samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle < -10); } } } catch (Exception e) { LogPrint.Exception(e); } } private void OnEvaluation() { if (_samplingYogaResult == null || _samplingYogaResult == false) { _samplingYogaResult = null; EventManager.Instance.Dispatch(YogaEventType.Action_Fail); return; } EventManager.Instance.Dispatch(YogaEventType.Action_Success); _comboTimes++; if (_comboTimes >= 3) { AudioManager.Instance.PlayCVInQueue("WellDone"); } else if (_comboTimes == 2) { AudioManager.Instance.PlayCVInQueue("Nice"); } else { AudioManager.Instance.PlayCVInQueue("Good"); } _samplingYogaResult = null; } /// extra private void ExtraMoveDistanceExactly() { AudioManager.Instance.PlayCVInQueue("ExactlyCorrect"); } private void ExtraMoveDistanceNotAccurate() { AudioManager.Instance.PlayCVInQueue("NotAccurate"); } private void ExtraSpeedTooFast() { AudioManager.Instance.PlayCVInQueue("SpeedTooFast"); } private void ExtraSpeedTooSlow() { AudioManager.Instance.PlayCVInQueue("SpeedTooSlow"); } #endregion public bool ActionCheckPoints(List points) { if (points == null || points.Count == 0) { LogPrint.Log("ActionCheckPoints points is null"); return false; } //for (int i = 0; i < points.Count; i++) //{ // Point p = points[i]; // if (p != new Point(-1, -1)) // LogPrint.Warning($"ActionPoints p({i}): {i.tagName()}, value: {p.x},{p.y}"); //} foreach (var p in LevelData.MustPoints) { if (!p.IsValid(points)) //有一个点不符合 返回false { //LogPrint.Log($"ActionCheckPoints failed, {p} is not valid"); return false; } } //不设的情况下,不检测 if (LevelData.AnyPoints != null && LevelData.AnyPoints.Count > 0) { foreach (var p in LevelData.AnyPoints) { if (p.IsValid(points)) //有一个点符合 返回true { return true; } } return false; //没有一个点符合 返回false } return true; } private Vector2 _baseNose = -Vector2.one; private Vector2 _baseNeck = -Vector2.one; private Vector2 _baseLEye = -Vector2.one; private Vector2 _baseREye = -Vector2.one; public void SetBasePosPoint() { //获取近1秒内所有关键点的平均值 var cacheCopy = new List<(DateTime, List)>(_estimateKeyPointsCache); var poinsList = cacheCopy.Where(x => x.Item1 >= DateTime.Now.AddSeconds(-1) && x.Item1 <= DateTime.Now).ToList(); _baseNose = getAveragePointVector(poinsList, "Nose"); _baseNeck = getAveragePointVector(poinsList, "Neck"); _baseLEye = getAveragePointVector(poinsList, "LEye"); _baseREye = getAveragePointVector(poinsList, "REye"); } private Vector2 getAveragePointVector(List<(DateTime, List)> poinsList, string partName) { int index = 0; Vector2 retVal = -Vector2.one; Vector2 totalVector = Vector2.zero; foreach (var item in poinsList) { Vector2 vector = partName.vector(item.Item2); if (vector != -Vector2.one) { totalVector += vector; index++; } } if (index != 0) { retVal = totalVector / index; } return retVal; } }