using OpenCVForUnity.CoreModule; using OpenCVForUnity.DnnModule; using OpenCVForUnity.ImgprocModule; using OpenCVForUnity.ObjdetectModule; using OpenCVForUnityExample.DnnModel; using System; using System.Collections.Generic; using System.Linq; using System.Runtime.InteropServices; using System.Text; using UnityEngine; public class MFacialExpressionRecognizer : FacialExpressionRecognizer { List mPalette = new List(); public MFacialExpressionRecognizer(string modelFilepath, string SF_modelFilepath, string SF_configFilepath, int backend = Dnn.DNN_BACKEND_OPENCV, int target = Dnn.DNN_TARGET_CPU):base(modelFilepath,SF_modelFilepath,SF_configFilepath,backend,target) { mPalette = new List(); mPalette.Add(new Scalar(255, 56, 56, 255)); mPalette.Add(new Scalar(82, 0, 133, 255)); mPalette.Add(new Scalar(52, 69, 147, 255)); mPalette.Add(new Scalar(255, 178, 29, 255)); mPalette.Add(new Scalar(55, 55, 55, 255)); mPalette.Add(new Scalar(100, 115, 255, 255)); mPalette.Add(new Scalar(255, 112, 31, 255)); } public FaceInfo[] visualize(Mat image, List results, Mat faces, bool isRGB = false) { if (image.IsDisposed) return new FaceInfo[0]; if (results.Count != faces.rows()) return new FaceInfo[0]; StringBuilder sb = null; //if (print_results) // sb = new StringBuilder(); FaceInfo[] faceInfos = new FaceInfo[results.Count]; for (int i = 0; i < results.Count; ++i) { float[] face_box = new float[4]; faces.get(i, 0, face_box); float left = face_box[0] + 2; float top = face_box[1] + 2; float right = face_box[0] + face_box[2] - 2; float bottom = face_box[1] + face_box[3] - 2; ClassificationData bmData = getBestMatchData(results[i]); int classId = (int)bmData.cls; string label = getClassLabel(bmData.cls) + ", " + String.Format("{0:0.0000}", bmData.conf); Scalar c = mPalette[classId % mPalette.Count]; Scalar color = isRGB ? c : new Scalar(c.val[2], c.val[1], c.val[0], c.val[3]); // draw box Imgproc.rectangle(image, new Point(left, top), new Point(right, bottom), color, 2); // draw label int[] baseLine = new int[1]; Size labelSize = Imgproc.getTextSize(label, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine); top = Mathf.Max((float)top, (float)labelSize.height); Imgproc.rectangle(image, new Point(left, top + 2), new Point(left + labelSize.width, top + labelSize.height + baseLine[0] + 2), color, Core.FILLED); Imgproc.putText(image, label, new Point(left, top + labelSize.height + 2), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, Scalar.all(255), 1, Imgproc.LINE_AA); FaceInfo info = new FaceInfo(); info.expression = getClassExp(bmData.cls); info.confidence = bmData.conf; info.box = new UnityEngine.Rect(left, bottom, right - left, top - bottom); faceInfos[i] = info; // Print results //if (print_results) //{ // sb.AppendLine(String.Format("-----------expression {0}-----------", i + 1)); //sb.AppendLine(String.Format("Best match: " + getClassLabel(bmData.cls) + ", " + bmData)); //} } //if (print_results) // Debug.Log(sb); return faceInfos; } public FaceInfo.Expression getClassExp(float id) { /* * ("angry"); ("disgust"); ("fearful"); ("happy"); ("neutral"); ("sad"); ("surprised"); * */ int index = (int)id; if(index >= 0) { switch (index) { case 0: return FaceInfo.Expression.Angry; case 1: return FaceInfo.Expression.Disgust; case 2: return FaceInfo.Expression.Fearful; case 3: return FaceInfo.Expression.Happy; case 4: return FaceInfo.Expression.Neutral; case 5: return FaceInfo.Expression.Sad; case 6: return FaceInfo.Expression.Suprised; default: return FaceInfo.Expression.Neutral; } } return FaceInfo.Expression.Neutral; } }