Health/Assets/Scripts/PoseCheck/MotionCaptureManager.cs

using OpenCVForUnity.CoreModule;
using OpenCVForUnity.DnnModule;
using OpenCVForUnity.ImgprocModule;
using OpenCVForUnity.UnityUtils;
using OpenCVForUnity.UnityUtils.Helper;
using OpenCVForUnityExample;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using UnityEngine;
using UnityEngine.Video;

namespace Yoga
{
    [RequireComponent(typeof(WebCamTextureToMatHelper))]
    public class MotionCaptureManager : MonoSingleton<MotionCaptureManager>
    {
        private WebCamTextureToMatHelper _webCamTextureToMatHelper;
        private bool isInited = false;
        private Texture2D texture;

        public float SamplingRate = 0.3f;

        private List<float[]> _voloResult = new List<float[]>();
        private List<Point> _currPersonPoints = new List<Point>();

        private Mat _bgrMat;

        private bool _isOnCamCapture = false;
        public bool IsOnCamCapture { get => _isOnCamCapture; internal set => _isOnCamCapture = value; }
        public WebCamTextureToMatHelper WebCamTextureToMatHelper => _webCamTextureToMatHelper;

        public List<Point> CurrPersonPoints { get => _currPersonPoints; set => _currPersonPoints = value; }
        public List<float[]> VoloResult { get => _voloResult; set => _voloResult = value; }
        private bool _isCorrectAction = false;
        private Mat _rgbaMat;
        public Mat RgbaMat { get => _rgbaMat; set => _rgbaMat = value; }

        private void OnEnable()
        {
            EventManager.Instance.AddEventListener(YogaEventType.StartMotionCapture, OnStartMotionCapture);
            EventManager.Instance.AddEventListener(YogaEventType.StopMotionCapture, OnStopMotionCapture);
            EventManager.Instance.AddEventListener(YogaEventType.EstimateAction, EstimateAction);
            EventManager.Instance.AddEventListener(YogaEventType.ScoreUpdate, ScoreUpdate);
            EventManager.Instance.AddEventListener(YogaEventType.GetActionBasePoint, GetActionBasePoint);
        }

        private void OnDisable()
        {
            EventManager.Instance.RemoveEventListener(YogaEventType.StartMotionCapture, OnStartMotionCapture);
            EventManager.Instance.RemoveEventListener(YogaEventType.StopMotionCapture, OnStopMotionCapture);
            EventManager.Instance.RemoveEventListener(YogaEventType.EstimateAction, EstimateAction);
            EventManager.Instance.RemoveEventListener(YogaEventType.ScoreUpdate, ScoreUpdate);
            EventManager.Instance.RemoveEventListener(YogaEventType.GetActionBasePoint, GetActionBasePoint);
        }

        public override void Init()
        {
            base.Init();

            _webCamTextureToMatHelper = gameObject.GetComponent<WebCamTextureToMatHelper>();
            _webCamTextureToMatHelper.Initialize();

            UIManager.Instance.LoadReset();
            
            if (YogaManager.Instance.Action == null)
                YogaManager.Instance.InitData();

            UIManager.Instance.ShowPanel<ActionGuideVideoPanel>(false, YogaManager.Instance.Action);

            Utils.setDebugMode(true); //打印日志
            //LoadModels();//加载模型

            switch (YogaManager.Instance.Action.ModelType)
            {
                case ModelType.OpenPose:
                    YogaManager.Instance.CurrentEstimator = new OpenPoseEsimater();
                    break;
                case ModelType.Mediapipe:
                    YogaManager.Instance.CurrentEstimator = new MediaPipeEstimator();
                    break;
            }
            YogaManager.Instance.CurrentEstimator.Init();

            CVEstimator.Instance.Init();//初始化姿态检测
        }

        private void Update()
        {
            if (!_isOnCamCapture)
            {
                return;
            }
            if (!transform.gameObject.activeSelf)
                transform.gameObject.SetActive(true);

            if (_webCamTextureToMatHelper.IsPlaying() && _webCamTextureToMatHelper.DidUpdateThisFrame())
            {
                Mat img = _webCamTextureToMatHelper.GetMat();
                Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB);
                _rgbaMat = img.clone();

                YogaManager.Instance.CurrentEstimator.Check(ref img);//检测模型，将错误信息打印在图片上


                if (_voloResult.Count >= 2)
                    YogaManager.Instance.CurrentEstimator.DebugPrintObjectLayout(img, _voloResult[0], _voloResult[1], _voloResult[2]);

                if (_currPersonPoints != null && _currPersonPoints.Count > 0)
                {
                    List<Point> points = _currPersonPoints;

                    for (int i = 0; i < YogaConfig.POSE_PAIRS.GetLength(0); i++)
                    {
                        string partFrom = YogaConfig.POSE_PAIRS[i, 0];
                        string partTo = YogaConfig.POSE_PAIRS[i, 1];

                        int idFrom = YogaConfig.BODY_PARTS[partFrom];
                        int idTo = YogaConfig.BODY_PARTS[partTo];

                        if (points[idFrom] == new Point(-1, -1) || points[idTo] == new Point(-1, -1))
                            continue;

                        if (points[idFrom] != null && points[idTo] != null)
                        {
                            Imgproc.line(img, points[idFrom], points[idTo], new Scalar(0, 255, 0), 3);
                            Imgproc.ellipse(img, points[idFrom], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED);
                            Imgproc.ellipse(img, points[idTo], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED);
                        }
                    }
                }

                //#endif
                Utils.matToTexture2D(img, texture);
            }
        }

        #region Event Func
        private void OnStartMotionCapture()
        {
            this.enabled = true;
            _isOnCamCapture = true;

            CVEstimator.Instance.StartEstimation();//开始姿态检测
        }

        private void OnStopMotionCapture()
        {
            this.enabled = false;
            _isOnCamCapture = false;
        }
        private void GetActionBasePoint()
        {
            var startTime = DateTime.Now;
            while (true)
            {
                if (_currPersonPoints != null && _currPersonPoints.Count != 0)
                {
                    YogaManager.Instance.Points = _currPersonPoints;
                    break;
                }

                if (startTime.AddMilliseconds(100) < DateTime.Now)
                {
                    Debug.LogError("GetActionBasePoint timeout");
                    break;
                }
            }
        }

        private void EstimateAction(params object[] args)
        {
            var type = args.FirstOrDefault();
            if (type == null)
            {
                Debug.LogError("EstimateAction type is null");
                return;
            }
            AvatarAction actionType = (AvatarAction)args.FirstOrDefault();


            //检测动作
            var startTime = DateTime.Now;

            while (true)
            {
                if (YogaManager.Instance.ActionCheckPoints(_currPersonPoints))
                {
                    _isCorrectAction = (_isCorrectAction || YogaManager.Instance.IsCorrectAction(_currPersonPoints, actionType));
                    break;
                }
                if (startTime.AddMilliseconds(100) < DateTime.Now)
                {
                    Debug.LogWarning("请摆正姿势");
                    Debug.LogWarning("EstimateAction timeout");
                    break;
                }
            }
        }

        #endregion



        /// <summary>
        /// Raises the destroy event.
        /// </summary>
        void OnDestroy()
        {
            _webCamTextureToMatHelper.Dispose();

            YogaManager.Instance.CurrentEstimator.Dispose();

            _bgrMat.Dispose();
            CVEstimator.Instance.Dispose();
        }

        public void OnWebCamTextureToMatHelperInitialized()
        {
            Debug.Log("OnWebCamTextureToMatHelperInitialized");

            Mat webCamTextureMat = _webCamTextureToMatHelper.GetMat();

            texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.RGB24, false);
            Utils.matToTexture2D(webCamTextureMat, texture);

            this.gameObject.GetComponent<Renderer>().material.mainTexture = texture;

            gameObject.transform.localScale = new Vector3(webCamTextureMat.cols() / 10, webCamTextureMat.rows() / 10, 1);
            Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);

            float width = webCamTextureMat.width();
            float height = webCamTextureMat.height();

            float widthScale = (float)Screen.width / width;
            float heightScale = (float)Screen.height / height;
            if (widthScale < heightScale)
            {
                Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2;
            }
            else
            {
                Camera.main.orthographicSize = height / 2;
            }

            _bgrMat = new Mat(webCamTextureMat.rows(), webCamTextureMat.cols(), CvType.CV_8UC3);
        }

        //event call
        public void OnWebCamTextureToMatHelperDisposed()
        {
            Debug.Log("OnWebCamTextureToMatHelperDisposed");

            if (_bgrMat != null)
                _bgrMat.Dispose();

            if (texture != null)
            {
                Texture2D.Destroy(texture);
                texture = null;
            }
        }

        public void ScoreUpdate()
        {

            if (_isCorrectAction)
            {
                EventManager.Instance.Dispatch(YogaEventType.ActionSuccess);
            }
            else
            {
                EventManager.Instance.Dispatch(YogaEventType.ActionFailed);
            }

            _isCorrectAction = false;//重置
        }
    }
}