using Rect = OpenCVCompact.Rect; using OpenCVCompact; using System; using UnityEngine; using OpenCVForUnity.UnityUtils.Helper; using System.Collections.Generic; [RequireComponent(typeof(OpenCVCompact.WebCamTextureToMatHelper))] public class FaceDetectManager : MonoBehaviour { //Related To DNN Model dnn.DNNUtils dnnUtils; //Image Capture OpenCVCompact.WebCamTextureToMatHelper _webCamTextureToMatHelper; Texture2D videoTexture; private double _lndmrkEstScore; private Mat mat4Process; private Mat grayMat4Process; private Mat mat4Display; private Mat lndmrk; private MatOfRect detectionResult; private Rect detectRect; int[] faceRect = new int[4]; private FaceDetectUI panel; // Start is called before the first frame update void Awake() { UIManager.Instance.LoadReset(); panel = UIManager.Instance.ShowPanel(); InitModel(); InitVariable(); _webCamTextureToMatHelper = gameObject.GetComponent(); _webCamTextureToMatHelper.Initialize(); } private void OnDestroy() { if (mat4Process != null) mat4Process.Dispose(); if (grayMat4Process != null) grayMat4Process.Dispose(); //if (img4Thread != null) // img4Thread.Dispose(); if (lndmrk != null) lndmrk.Dispose(); //if (lndmrk4Thread != null) // lndmrk4Thread.Dispose(); if (mat4Display != null) mat4Display.Dispose(); //if (mat4DisplayTexture != null) // mat4DisplayTexture.Dispose(); //if (probExp != null) // probExp.Dispose(); _webCamTextureToMatHelper.Dispose(); } private void InitVariable() { mat4Process = new Mat(); grayMat4Process = new Mat(); detectionResult = new MatOfRect(); detectRect = new Rect(); lndmrk = new Mat(2, 84, CvType.CV_32FC1, 0.0f); InvokeRepeating("AnalyzingFace", 0.5f, 1f); } private bool InitModel() { dnnUtils = new dnn.DNNUtils(); string dnnLndmrkDetectModelFilePath; string dnnFaceAttr7ModelFilePath; string dnnFaceExpModelFilePath; string multiTinyFaceDetectModelFilePath; string YOLOV3DetectModelFilePath; string YOLOV3DetectParamFilePath; string YOLOV3DetectOpenCVModelFilePath; string YOLOV3DetectOpenCVParamFilePath; //Init Model File Path #if (UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR dnnLndmrkDetectModelFilePath = Utils.getFilePath("FaceAnalyzer/face_lndmrk_detect_mobile.bin"); #else dnnLndmrkDetectModelFilePath = Utils.getFilePath("FaceAnalyzer/face_lndmrk_detect.bin"); #endif #if (UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR dnnFaceAttr7ModelFilePath = Utils.getFilePath("FaceAnalyzer/att_7_mobile.bin"); #else dnnFaceAttr7ModelFilePath = Utils.getFilePath("FaceAnalyzer/att_7.bin"); #endif #if (UNITY_ANDROID || UNITY_IOS) && !UNITY_EDITOR dnnFaceExpModelFilePath = Utils.getFilePath("FaceAnalyzer/face_exp_mobile.bin"); #else dnnFaceExpModelFilePath = Utils.getFilePath("FaceAnalyzer/face_exp.bin"); #endif multiTinyFaceDetectModelFilePath = Utils.getFilePath("FaceAnalyzer/multi_tiny_face_detect_mobile.bin"); YOLOV3DetectModelFilePath = Utils.getFilePath("FaceAnalyzer/yolov3.bin"); YOLOV3DetectParamFilePath = Utils.getFilePath("FaceAnalyzer/yolov3.param"); YOLOV3DetectOpenCVModelFilePath = Utils.getFilePath("FaceAnalyzer/yolov3-tiny.weights"); YOLOV3DetectOpenCVParamFilePath = Utils.getFilePath("FaceAnalyzer/yolov3-tiny.cfg"); bool initFaceLndmrkRes; initFaceLndmrkRes = dnnUtils.InitFaceLandmarkDetect(dnnLndmrkDetectModelFilePath); if (initFaceLndmrkRes == false) { Debug.LogError(dnnLndmrkDetectModelFilePath + " file is not loaded."); Debug.LogError(dnnLndmrkDetectModelFilePath + " file is not existed on StreamingAssets Folder. Please copy from ˇ°Assets/FaceAnalyzer/StreamingAssets/ˇ± to ˇ°Assets/StreamingAssets/ˇ± folder."); } dnnUtils.InitHeadPoseEstimation(); int initFaceAttrRes = dnnUtils.InitFaceAttribNet_7(dnnFaceAttr7ModelFilePath); if (initFaceAttrRes <= 0) { Debug.LogError(dnnFaceAttr7ModelFilePath + " file is not loaded."); Debug.LogError(dnnFaceAttr7ModelFilePath + " file is not existed on StreamingAssets Folder. Please copy from ˇ°Assets/FaceAnalyzer/StreamingAssets/ˇ± to ˇ°Assets/StreamingAssets/ˇ± folder."); } int initFaceExpRes = dnnUtils.InitFaceExpressionNet_7(dnnFaceExpModelFilePath); if (initFaceExpRes <= 0) { Debug.LogError(dnnFaceExpModelFilePath + " file is not loaded."); Debug.LogError(dnnFaceExpModelFilePath + " file is not existed on StreamingAssets Folder. Please copy from ˇ°Assets/FaceAnalyzer/StreamingAssets/ˇ± to ˇ°Assets/StreamingAssets/ˇ± folder."); } int initMultiTinyFaceDetectRes = dnnUtils.InitMultiTinyFaceDetector(multiTinyFaceDetectModelFilePath); if (initMultiTinyFaceDetectRes <= 0) { Debug.LogError(multiTinyFaceDetectModelFilePath + " file is not loaded."); Debug.LogError(multiTinyFaceDetectModelFilePath + " file is not existed on StreamingAssets Folder. Please copy from ˇ°Assets/FaceAnalyzer/StreamingAssets/ˇ± to ˇ°Assets/StreamingAssets/ˇ± folder."); } int initYOLOV3DetectRes = dnnUtils.InitYOLOV3Detector(YOLOV3DetectModelFilePath, YOLOV3DetectParamFilePath); if (initYOLOV3DetectRes == -1) { Debug.LogError(YOLOV3DetectModelFilePath + " file is not loaded."); Debug.LogError(YOLOV3DetectModelFilePath + " file is not existed on StreamingAssets Folder. Please copy from ˇ°Assets/FaceAnalyzer/StreamingAssets/ˇ± to ˇ°Assets/StreamingAssets/ˇ± folder."); } else if (initYOLOV3DetectRes == -2) { Debug.LogError(YOLOV3DetectParamFilePath + " file is not loaded."); Debug.LogError(YOLOV3DetectParamFilePath + " file is not existed on StreamingAssets Folder. Please copy from ˇ°Assets/FaceAnalyzer/StreamingAssets/ˇ± to ˇ°Assets/StreamingAssets/ˇ± folder."); } dnnUtils.InitLabels_7(); dnnUtils.InitExpLabels_7(); dnnUtils.InitObjectLabels_21(); dnnUtils.InitKalmanFilter(10f); return true; } private void AnalyzingFace() { if (!_webCamTextureToMatHelper.IsPlaying() || !_webCamTextureToMatHelper.DidUpdateThisFrame()) return; mat4Display = _webCamTextureToMatHelper.GetMat(); mat4Display.copyTo(mat4Process); //Face Detect panel.RefreshData(LandmarkDetect()); } private Dictionary LandmarkDetect() { Mat prob = new Mat(); Dictionary retVal = null; if (dnnUtils.EstFaceAttribNet_7(mat4Process.nativeObj, lndmrk.nativeObj, CvType.COLOR_RGBA, prob.nativeObj) == true) { dnnUtils.ParseEstFaceAttrib_7(prob); string text = string.Empty; retVal = new Dictionary(dnnUtils.dnnFaceAttribRes_7); int idx = 0; foreach (var pair in dnnUtils.dnnFaceAttribRes_7) { string tmp; if (idx % 2 == 0) tmp = string.Format("[{0}] : {1}\n", pair.Key, pair.Value); else tmp = string.Format("[{0}] : {1}\t\t", pair.Key, pair.Value); text = string.Concat(text, tmp); idx = idx + 1; } Debug.LogWarning(text.Substring(0, text.Length - 1)); } prob.Dispose(); return retVal; } }