using OpenCVForUnity.CoreModule; using OpenCVForUnity.ImgprocModule; using OpenCVForUnity.UnityUtils; using OpenCVForUnityExample.DnnModel; using System; using System.Collections.Generic; using Unity.Mathematics; using UnityEngine; using UnityEngine.Experimental.GlobalIllumination; using static UnityEngine.Networking.UnityWebRequest; using OpenCVRange = OpenCVForUnity.CoreModule.Range; public class MediaPipeEstimator : Estimator { private MediaPipePersonDetector _personDetector; private MediaPipePoseEstimator _poseEstimator; public override void InitModel() { var personDetectModelPath = Utils.getFilePath(YogaConfig.MODEL_PATHS[ModelType.MediapipePersonDetect]); if (string.IsNullOrEmpty(personDetectModelPath)) { LogPrint.Error(YogaConfig.MODEL_PATHS[ModelType.MediapipePersonDetect] + " is not loaded. Please read “StreamingAssets/OpenCVForUnity/dnn/setup_dnn_module.pdf” to make the necessary setup."); } else { _personDetector = new MediaPipePersonDetector(personDetectModelPath, 0.3f, 0.6f, 5000); } var poseModelPath = Utils.getFilePath(YogaConfig.MODEL_PATHS[ModelType.MediapipePose]); if (string.IsNullOrEmpty(poseModelPath)) { LogPrint.Error(YogaConfig.MODEL_PATHS[ModelType.MediapipePose] + " is not loaded. Please read “StreamingAssets/OpenCVForUnity/dnn/setup_dnn_module.pdf” to make the necessary setup."); } else { _poseEstimator = new MediaPipePoseEstimator(poseModelPath, 0.9f); } } public override bool Check(ref Mat img) { if (_personDetector == null || _poseEstimator == null) { Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255, 255), 2, Imgproc.LINE_AA, false); return false; } return true; } public override bool Esitmate(Mat bgrMat, Mat rgbaMat, out List points) { points = null; Mat results = _personDetector.infer(bgrMat); if (results.rows() == 0) return false; float imgSize = Mathf.Max((float)bgrMat.size().width, (float)bgrMat.size().height); float x_factor = 1; float y_factor = 1; float x_shift = (imgSize - (float)bgrMat.size().width) / 2f; float y_shift = (imgSize - (float)bgrMat.size().height) / 2f; float maxSize = 30; int maxRectIndex = -1; for (int i = 0; i < results.rows(); ++i) { float[] results_arr = new float[4]; results.get(i, 0, results_arr); float x1 = Mathf.Floor(results_arr[0] * x_factor - x_shift); float y1 = Mathf.Floor(results_arr[1] * y_factor - y_shift); float x2 = Mathf.Floor(results_arr[2] * x_factor - x_shift); float y2 = Mathf.Floor(results_arr[3] * y_factor - y_shift); //主副驾 if (!GlobalData.Instance.IsDriverPosition) { if ((x1 + x2) / 2 < bgrMat.size().width / 2) //副驾,主驾位跳过 { continue; } } //if (GlobalData.Instance.Position == PositionType.Driver) else { if ((x1 + x2) / 2 >= bgrMat.size().width / 2) //主驾,副驾位跳过 { continue; } } float size = Mathf.Abs(x2 - x1) * Mathf.Abs(y2 - y1); if (maxSize < size) { maxSize = size; maxRectIndex = i; } float[] box = new float[4] { x1, y1, x2, y2 }; YogaManager.Instance.PersonRectResult = new List { box }; } if (maxRectIndex < 0) //没有找到合适 { LogPrint.Log("没有找到合适的框"); return false; } List result = _poseEstimator.infer(bgrMat, results.row(maxRectIndex)); //_poseEstimator.visualize(rgbaMat, results[0], false, false); points = GetKeypoints(result[0]); //没有检测到人体 return false; } public override void DebugPrint(ref Mat img, bool isRGB = false) { } private List GetKeypoints(Mat result) { var retVal = new List(); if (result.empty() || result.rows() < 317) return retVal; //获取关键点 float[] conf = new float[1]; result.get(316, 0, conf); float[] bbox = new float[4]; result.get(0, 0, bbox); int auxiliary_points_num = 6; Mat results_col4_199_39x5 = result.rowRange(new OpenCVRange(4, 199 - (5 * auxiliary_points_num))).reshape(1, 39 - auxiliary_points_num); float[] landmarks_screen_xy = new float[(39 - auxiliary_points_num) * 2]; results_col4_199_39x5.colRange(new OpenCVRange(0, 2)).get(0, 0, landmarks_screen_xy); float[] landmarks_screen_xyz = new float[(39 - auxiliary_points_num) * 3]; results_col4_199_39x5.colRange(new OpenCVRange(0, 3)).get(0, 0, landmarks_screen_xyz); // # only show visible keypoints which presence bigger than 0.8 float[] landmarks_presence = new float[(39 - auxiliary_points_num)]; results_col4_199_39x5.colRange(new OpenCVRange(4, 5)).get(0, 0, landmarks_presence); Mat results_col199_316_39x3 = result.rowRange(new OpenCVRange(199, 316 - (3 * auxiliary_points_num))).reshape(1, 39 - auxiliary_points_num); float[] landmarks_world = new float[(39 - auxiliary_points_num) * 3]; results_col199_316_39x3.get(0, 0, landmarks_world); //将关键点映射到现有的open pose关键点上 retVal.Add(GetPointData(landmarks_screen_xy, landmarks_presence, 0)); //Nose retVal.Add(new Point(-1, -1)); //Neck retVal.Add(GetPointData(landmarks_screen_xy, landmarks_presence, 11)); //LShoulder retVal.Add(GetPointData(landmarks_screen_xy, landmarks_presence, 13)); //LElbow retVal.Add(GetPointData(landmarks_screen_xy, landmarks_presence, 15)); //LWrist retVal.Add(GetPointData(landmarks_screen_xy, landmarks_presence, 12)); //RShoulder retVal.Add(GetPointData(landmarks_screen_xy, landmarks_presence, 14)); //RElbow retVal.Add(GetPointData(landmarks_screen_xy, landmarks_presence, 16)); //RWrist retVal.Add(GetPointData(landmarks_screen_xy, landmarks_presence, 23)); //LHip retVal.Add(GetPointData(landmarks_screen_xy, landmarks_presence, 25)); //LKnee retVal.Add(GetPointData(landmarks_screen_xy, landmarks_presence, 27)); //LAnkle retVal.Add(GetPointData(landmarks_screen_xy, landmarks_presence, 24)); //RHip retVal.Add(GetPointData(landmarks_screen_xy, landmarks_presence, 26)); //RKnee retVal.Add(GetPointData(landmarks_screen_xy, landmarks_presence, 28)); //RAnkle retVal.Add(GetPointData(landmarks_screen_xy, landmarks_presence, 2)); //LEye retVal.Add(GetPointData(landmarks_screen_xy, landmarks_presence, 5)); //REye retVal.Add(GetPointData(landmarks_screen_xy, landmarks_presence, 7)); //LEar retVal.Add(GetPointData(landmarks_screen_xy, landmarks_presence, 8)); //REar retVal.Add(new Point(-1, -1)); //Background return retVal; } private Point GetPointData(float[] landmarks, float[] _landmarks_presence, int index) { if (_landmarks_presence[index] < 0.8f) return null; index = index * 2; return new Point(landmarks[index], landmarks[index + 1]); } public override void DisposeModel() { } }