using OpenCVForUnity.CoreModule; using OpenCVForUnity.DnnModule; using OpenCVForUnity.ImgprocModule; using OpenCVForUnity.UnityUtils; using OpenCVForUnity.UnityUtils.Helper; using OpenCVForUnityExample; using System; using System.Collections; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using UnityEngine; using UnityEngine.Video; namespace Yoga { [RequireComponent(typeof(WebCamTextureToMatHelper))] public class MotionCaptureManager : MonoSingleton { private WebCamTextureToMatHelper _webCamTextureToMatHelper; private bool isInited = false; private Texture2D texture; public float SamplingRate = 0.3f; private List _voloResult = new List(); private List _currPersonPoints = new List(); private Mat _bgrMat; private bool _isOnCamCapture = false; public bool IsOnCamCapture { get => _isOnCamCapture; internal set => _isOnCamCapture = value; } public WebCamTextureToMatHelper WebCamTextureToMatHelper => _webCamTextureToMatHelper; public List CurrPersonPoints { get => _currPersonPoints; set => _currPersonPoints = value; } public List VoloResult { get => _voloResult; set => _voloResult = value; } private bool _isCorrectAction = false; private Mat _rgbaMat; public Mat RgbaMat { get => _rgbaMat; set => _rgbaMat = value; } private void OnEnable() { EventManager.Instance.AddEventListener(YogaEventType.StartMotionCapture, OnStartMotionCapture); EventManager.Instance.AddEventListener(YogaEventType.StopMotionCapture, OnStopMotionCapture); EventManager.Instance.AddEventListener(YogaEventType.EstimateAction, EstimateAction); EventManager.Instance.AddEventListener(YogaEventType.ScoreUpdate, ScoreUpdate); EventManager.Instance.AddEventListener(YogaEventType.GetActionBasePoint, GetActionBasePoint); } private void OnDisable() { EventManager.Instance.RemoveEventListener(YogaEventType.StartMotionCapture, OnStartMotionCapture); EventManager.Instance.RemoveEventListener(YogaEventType.StopMotionCapture, OnStopMotionCapture); EventManager.Instance.RemoveEventListener(YogaEventType.EstimateAction, EstimateAction); EventManager.Instance.RemoveEventListener(YogaEventType.ScoreUpdate, ScoreUpdate); EventManager.Instance.RemoveEventListener(YogaEventType.GetActionBasePoint, GetActionBasePoint); } public override void Init() { base.Init(); _webCamTextureToMatHelper = gameObject.GetComponent(); _webCamTextureToMatHelper.Initialize(); UIManager.Instance.LoadReset(); if (YogaManager.Instance.Action == null) YogaManager.Instance.InitData(); UIManager.Instance.ShowPanel(false, YogaManager.Instance.Action); Utils.setDebugMode(true); //打印日志 //LoadModels();//加载模型 switch (YogaManager.Instance.Action.ModelType) { case ModelType.OpenPose: YogaManager.Instance.CurrentEstimator = new OpenPoseEsimater(); break; case ModelType.Mediapipe: YogaManager.Instance.CurrentEstimator = new MediaPipeEstimator(); break; } YogaManager.Instance.CurrentEstimator.Init(); CVEstimator.Instance.Init();//初始化姿态检测 } private void Update() { if (!_isOnCamCapture) { return; } if (!transform.gameObject.activeSelf) transform.gameObject.SetActive(true); if (_webCamTextureToMatHelper.IsPlaying() && _webCamTextureToMatHelper.DidUpdateThisFrame()) { Mat img = _webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB); _rgbaMat = img.clone(); YogaManager.Instance.CurrentEstimator.Check(ref img);//检测模型,将错误信息打印在图片上 if (_voloResult.Count >= 2) YogaManager.Instance.CurrentEstimator.DebugPrintObjectLayout(img, _voloResult[0], _voloResult[1], _voloResult[2]); if (_currPersonPoints != null && _currPersonPoints.Count > 0) { List points = _currPersonPoints; for (int i = 0; i < YogaConfig.POSE_PAIRS.GetLength(0); i++) { string partFrom = YogaConfig.POSE_PAIRS[i, 0]; string partTo = YogaConfig.POSE_PAIRS[i, 1]; int idFrom = YogaConfig.BODY_PARTS[partFrom]; int idTo = YogaConfig.BODY_PARTS[partTo]; if (points[idFrom] == new Point(-1, -1) || points[idTo] == new Point(-1, -1)) continue; if (points[idFrom] != null && points[idTo] != null) { Imgproc.line(img, points[idFrom], points[idTo], new Scalar(0, 255, 0), 3); Imgproc.ellipse(img, points[idFrom], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED); Imgproc.ellipse(img, points[idTo], new Size(3, 3), 0, 0, 360, new Scalar(0, 0, 255), Core.FILLED); } } } //#endif Utils.matToTexture2D(img, texture); } } #region Event Func private void OnStartMotionCapture() { this.enabled = true; _isOnCamCapture = true; CVEstimator.Instance.StartEstimation();//开始姿态检测 } private void OnStopMotionCapture() { this.enabled = false; _isOnCamCapture = false; } private void GetActionBasePoint() { var startTime = DateTime.Now; while (true) { if (_currPersonPoints != null && _currPersonPoints.Count != 0) { YogaManager.Instance.Points = _currPersonPoints; break; } if (startTime.AddMilliseconds(100) < DateTime.Now) { Debug.LogError("GetActionBasePoint timeout"); break; } } } private void EstimateAction(params object[] args) { var type = args.FirstOrDefault(); if (type == null) { Debug.LogError("EstimateAction type is null"); return; } AvatarAction actionType = (AvatarAction)args.FirstOrDefault(); //检测动作 var startTime = DateTime.Now; while (true) { if (YogaManager.Instance.ActionCheckPoints(_currPersonPoints)) { _isCorrectAction = (_isCorrectAction || YogaManager.Instance.IsCorrectAction(_currPersonPoints, actionType)); break; } if (startTime.AddMilliseconds(100) < DateTime.Now) { Debug.LogWarning("请摆正姿势"); Debug.LogWarning("EstimateAction timeout"); break; } } } #endregion /// /// Raises the destroy event. /// void OnDestroy() { _webCamTextureToMatHelper.Dispose(); YogaManager.Instance.CurrentEstimator.Dispose(); _bgrMat.Dispose(); CVEstimator.Instance.Dispose(); } public void OnWebCamTextureToMatHelperInitialized() { Debug.Log("OnWebCamTextureToMatHelperInitialized"); Mat webCamTextureMat = _webCamTextureToMatHelper.GetMat(); texture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.RGB24, false); Utils.matToTexture2D(webCamTextureMat, texture); this.gameObject.GetComponent().material.mainTexture = texture; gameObject.transform.localScale = new Vector3(webCamTextureMat.cols() / 10, webCamTextureMat.rows() / 10, 1); Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); float width = webCamTextureMat.width(); float height = webCamTextureMat.height(); float widthScale = (float)Screen.width / width; float heightScale = (float)Screen.height / height; if (widthScale < heightScale) { Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2; } else { Camera.main.orthographicSize = height / 2; } _bgrMat = new Mat(webCamTextureMat.rows(), webCamTextureMat.cols(), CvType.CV_8UC3); } //event call public void OnWebCamTextureToMatHelperDisposed() { Debug.Log("OnWebCamTextureToMatHelperDisposed"); if (_bgrMat != null) _bgrMat.Dispose(); if (texture != null) { Texture2D.Destroy(texture); texture = null; } } public void ScoreUpdate() { if (_isCorrectAction) { EventManager.Instance.Dispatch(YogaEventType.ActionSuccess); } else { EventManager.Instance.Dispatch(YogaEventType.ActionFailed); } _isCorrectAction = false;//重置 } } }