diff --git a/Assets/Resources/Animation/SitUpright.anim b/Assets/Resources/Animation/SitUpright.anim index 4a99640..4233b96 100644 --- a/Assets/Resources/Animation/SitUpright.anim +++ b/Assets/Resources/Animation/SitUpright.anim @@ -29872,3 +29872,10 @@ AnimationClip: floatParameter: 0 intParameter: 0 messageOptions: 0 + - time: 5.233333 + functionName: SetBasePosPoint + data: + objectReferenceParameter: {fileID: 0} + floatParameter: 0 + intParameter: 0 + messageOptions: 0 diff --git a/Assets/Resources/UI/ActionListUI.prefab b/Assets/Resources/UI/ActionListUI.prefab index fd531a8..16979f6 100644 --- a/Assets/Resources/UI/ActionListUI.prefab +++ b/Assets/Resources/UI/ActionListUI.prefab @@ -989,7 +989,8 @@ MonoBehaviour: m_EditorClassIdentifier: backgroundImage: {fileID: 21300000, guid: fa5ad33758d1e094b91bd2a227631ec5, type: 3} buttonTitle: Meditation - buttonDescription: DESCRIPTION + buttonDescription: Spiritual Yoga, which can relieving mental stress, enhance your + focus useCustomResources: 0 enableStatus: 1 statusItem: 0 @@ -1579,8 +1580,8 @@ RectTransform: m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 0, y: 0} - m_AnchoredPosition: {x: 0, y: 0} - m_SizeDelta: {x: 0, y: 0} + m_AnchoredPosition: {x: 125, y: 0} + m_SizeDelta: {x: 200, y: 0} m_Pivot: {x: 0.5, y: 0.5} --- !u!222 &3299300514319254663 CanvasRenderer: @@ -3998,8 +3999,8 @@ RectTransform: m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} m_AnchorMax: {x: 0, y: 0} - m_AnchoredPosition: {x: 0, y: 0} - m_SizeDelta: {x: 0, y: 0} + m_AnchoredPosition: {x: 125, y: 0} + m_SizeDelta: {x: 200, y: 0} m_Pivot: {x: 0.5, y: 0.5} --- !u!222 &1105460671971188788 CanvasRenderer: diff --git a/Assets/Resources/Video/Action1.mp4 b/Assets/Resources/Video/Action1.mp4 index 20adb32..79a18f3 100644 Binary files a/Assets/Resources/Video/Action1.mp4 and b/Assets/Resources/Video/Action1.mp4 differ diff --git a/Assets/Resources/Video/Action2.mp4 b/Assets/Resources/Video/Action2.mp4 index 5e9fbb4..41920b5 100644 Binary files a/Assets/Resources/Video/Action2.mp4 and b/Assets/Resources/Video/Action2.mp4 differ diff --git a/Assets/Resources/Video/Action3.mp4 b/Assets/Resources/Video/Action3.mp4 index a08bb57..63a66b5 100644 Binary files a/Assets/Resources/Video/Action3.mp4 and b/Assets/Resources/Video/Action3.mp4 differ diff --git a/Assets/Scenes/FaceDetect.unity b/Assets/Scenes/FaceDetect.unity index 89ad592..ea69b13 100644 --- a/Assets/Scenes/FaceDetect.unity +++ b/Assets/Scenes/FaceDetect.unity @@ -449,8 +449,7 @@ MonoBehaviour: m_RequiresDepthTextureOption: 2 m_RequiresOpaqueTextureOption: 2 m_CameraType: 0 - m_Cameras: - - {fileID: 1969623179} + m_Cameras: [] m_RendererIndex: -1 m_VolumeLayerMask: serializedVersion: 2 @@ -478,6 +477,53 @@ MonoBehaviour: mipBias: 0 varianceClampScale: 0.9 contrastAdaptiveSharpening: 0 +--- !u!1 &709787121 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 709787123} + - component: {fileID: 709787122} + m_Layer: 0 + m_Name: USBFaceDetectManager + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 0 +--- !u!114 &709787122 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 709787121} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: b1a3af86bea67944b9cce9661ba1bfc6, type: 3} + m_Name: + m_EditorClassIdentifier: + image: {fileID: 1878964050} + cutImage: {fileID: 1838964607} + UVCFilters: [] +--- !u!4 &709787123 +Transform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 709787121} + serializedVersion: 2 + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: -0.25406203, y: 2994.0242, z: -0.2102798} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 0} + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} --- !u!1 &1014002332 GameObject: m_ObjectHideFlags: 0 @@ -491,7 +537,7 @@ GameObject: - component: {fileID: 1014002335} - component: {fileID: 1014002334} - component: {fileID: 1014002339} - - component: {fileID: 1014002338} + - component: {fileID: 1014002340} m_Layer: 0 m_Name: FaceDetectManager m_TagString: Untagged @@ -586,18 +632,6 @@ Transform: m_Children: [] m_Father: {fileID: 0} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} ---- !u!114 &1014002338 -MonoBehaviour: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 1014002332} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: 3d8102f8264d54f4099a5fd817383837, type: 3} - m_Name: - m_EditorClassIdentifier: --- !u!114 &1014002339 MonoBehaviour: m_ObjectHideFlags: 0 @@ -622,8 +656,8 @@ MonoBehaviour: onInitialized: m_PersistentCalls: m_Calls: - - m_Target: {fileID: 1014002338} - m_TargetAssemblyTypeName: FaceDetectManager, Assembly-CSharp + - m_Target: {fileID: 1014002340} + m_TargetAssemblyTypeName: WebCamFaceDetectManager, Assembly-CSharp m_MethodName: OnWebCamTextureToMatHelperInitialized m_Mode: 1 m_Arguments: @@ -637,8 +671,8 @@ MonoBehaviour: onDisposed: m_PersistentCalls: m_Calls: - - m_Target: {fileID: 1014002338} - m_TargetAssemblyTypeName: FaceDetectManager, Assembly-CSharp + - m_Target: {fileID: 1014002340} + m_TargetAssemblyTypeName: WebCamFaceDetectManager, Assembly-CSharp m_MethodName: OnWebCamTextureToMatHelperDisposed m_Mode: 1 m_Arguments: @@ -652,8 +686,8 @@ MonoBehaviour: onErrorOccurred: m_PersistentCalls: m_Calls: - - m_Target: {fileID: 1014002338} - m_TargetAssemblyTypeName: FaceDetectManager, Assembly-CSharp + - m_Target: {fileID: 1014002340} + m_TargetAssemblyTypeName: WebCamFaceDetectManager, Assembly-CSharp m_MethodName: OnWebCamTextureToMatHelperErrorOccurred m_Mode: 0 m_Arguments: @@ -665,6 +699,20 @@ MonoBehaviour: m_BoolArgument: 0 m_CallState: 2 avoidAndroidFrontCameraLowLightIssue: 0 +--- !u!114 &1014002340 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1014002332} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 23c90b7e25f07ce4298ce424fedeb5e4, type: 3} + m_Name: + m_EditorClassIdentifier: + image: {fileID: 1878964050} + cutImage: {fileID: 1838964607} --- !u!1 &1359769552 GameObject: m_ObjectHideFlags: 0 @@ -760,6 +808,8 @@ RectTransform: m_ConstrainProportionsScale: 0 m_Children: - {fileID: 288140066} + - {fileID: 1878964052} + - {fileID: 1838964606} m_Father: {fileID: 0} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} @@ -767,7 +817,7 @@ RectTransform: m_AnchoredPosition: {x: 0, y: 0} m_SizeDelta: {x: 0, y: 0} m_Pivot: {x: 0, y: 0} ---- !u!1 &1969623176 +--- !u!1 &1838964605 GameObject: m_ObjectHideFlags: 0 m_CorrespondingSourceObject: {fileID: 0} @@ -775,135 +825,142 @@ GameObject: m_PrefabAsset: {fileID: 0} serializedVersion: 6 m_Component: - - component: {fileID: 1969623180} - - component: {fileID: 1969623179} - - component: {fileID: 1969623178} - - component: {fileID: 1969623177} - m_Layer: 0 - m_Name: DebugCamera + - component: {fileID: 1838964606} + - component: {fileID: 1838964608} + - component: {fileID: 1838964607} + m_Layer: 5 + m_Name: 'CutRawImage ' m_TagString: Untagged m_Icon: {fileID: 0} m_NavMeshLayer: 0 m_StaticEditorFlags: 0 - m_IsActive: 0 ---- !u!114 &1969623177 + m_IsActive: 1 +--- !u!224 &1838964606 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1838964605} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 1359769556} + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0, y: 0} + m_AnchorMax: {x: 1, y: 1} + m_AnchoredPosition: {x: 353.00073, y: -240} + m_SizeDelta: {x: -1515.9985, y: -520} + m_Pivot: {x: 0.5, y: 0.5} +--- !u!114 &1838964607 MonoBehaviour: m_ObjectHideFlags: 0 m_CorrespondingSourceObject: {fileID: 0} m_PrefabInstance: {fileID: 0} m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 1969623176} + m_GameObject: {fileID: 1838964605} m_Enabled: 1 m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: a79441f348de89743a2939f4d699eac1, type: 3} + m_Script: {fileID: 11500000, guid: 1344c3c82d62a2a41a3576d8abb8e3ea, type: 3} m_Name: m_EditorClassIdentifier: - m_RenderShadows: 1 - m_RequiresDepthTextureOption: 2 - m_RequiresOpaqueTextureOption: 2 - m_CameraType: 1 - m_Cameras: [] - m_RendererIndex: -1 - m_VolumeLayerMask: - serializedVersion: 2 - m_Bits: 1 - m_VolumeTrigger: {fileID: 0} - m_VolumeFrameworkUpdateModeOption: 2 - m_RenderPostProcessing: 0 - m_Antialiasing: 0 - m_AntialiasingQuality: 2 - m_StopNaN: 0 - m_Dithering: 0 - m_ClearDepth: 1 - m_AllowXRRendering: 1 - m_AllowHDROutput: 1 - m_UseScreenCoordOverride: 0 - m_ScreenSizeOverride: {x: 0, y: 0, z: 0, w: 0} - m_ScreenCoordScaleBias: {x: 0, y: 0, z: 0, w: 0} - m_RequiresDepthTexture: 0 - m_RequiresColorTexture: 0 - m_Version: 2 - m_TaaSettings: - quality: 3 - frameInfluence: 0.1 - jitterScale: 1 - mipBias: 0 - varianceClampScale: 0.9 - contrastAdaptiveSharpening: 0 ---- !u!81 &1969623178 -AudioListener: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 1969623176} - m_Enabled: 1 ---- !u!20 &1969623179 -Camera: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 1969623176} - m_Enabled: 1 - serializedVersion: 2 - m_ClearFlags: 4 - m_BackGroundColor: {r: 0.19215687, g: 0.3019608, b: 0.4745098, a: 0} - m_projectionMatrixMode: 1 - m_GateFitMode: 2 - m_FOVAxisMode: 0 - m_Iso: 200 - m_ShutterSpeed: 0.005 - m_Aperture: 16 - m_FocusDistance: 10 - m_FocalLength: 50 - m_BladeCount: 5 - m_Curvature: {x: 2, y: 11} - m_BarrelClipping: 0.25 - m_Anamorphism: 0 - m_SensorSize: {x: 36, y: 24} - m_LensShift: {x: 0, y: 0} - m_NormalizedViewPortRect: + m_Material: {fileID: 0} + m_Color: {r: 1, g: 1, b: 1, a: 1} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_Texture: {fileID: 0} + m_UVRect: serializedVersion: 2 x: 0 y: 0 width: 1 height: 1 - near clip plane: 0.3 - far clip plane: 500 - field of view: 23 - orthographic: 0 - orthographic size: 5 - m_Depth: 0 - m_CullingMask: - serializedVersion: 2 - m_Bits: 1 - m_RenderingPath: -1 - m_TargetTexture: {fileID: 0} - m_TargetDisplay: 0 - m_TargetEye: 3 - m_HDR: 1 - m_AllowMSAA: 1 - m_AllowDynamicResolution: 0 - m_ForceIntoRT: 0 - m_OcclusionCulling: 1 - m_StereoConvergence: 10 - m_StereoSeparation: 0.022 ---- !u!4 &1969623180 -Transform: +--- !u!222 &1838964608 +CanvasRenderer: m_ObjectHideFlags: 0 m_CorrespondingSourceObject: {fileID: 0} m_PrefabInstance: {fileID: 0} m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 1969623176} - serializedVersion: 2 + m_GameObject: {fileID: 1838964605} + m_CullTransparentMesh: 1 +--- !u!1 &1878964049 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 1878964052} + - component: {fileID: 1878964051} + - component: {fileID: 1878964050} + m_Layer: 5 + m_Name: RawImage + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!114 &1878964050 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1878964049} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 1344c3c82d62a2a41a3576d8abb8e3ea, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 1, g: 1, b: 1, a: 1} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_Texture: {fileID: 0} + m_UVRect: + serializedVersion: 2 + x: 0 + y: 0 + width: 1 + height: 1 +--- !u!222 &1878964051 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1878964049} + m_CullTransparentMesh: 1 +--- !u!224 &1878964052 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 1878964049} m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} - m_LocalPosition: {x: -3604, y: 481, z: 699} + m_LocalPosition: {x: 0, y: 0, z: 0} m_LocalScale: {x: 1, y: 1, z: 1} m_ConstrainProportionsScale: 0 m_Children: [] - m_Father: {fileID: 0} + m_Father: {fileID: 1359769556} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0, y: 0} + m_AnchorMax: {x: 1, y: 1} + m_AnchoredPosition: {x: -529.30554, y: -240} + m_SizeDelta: {x: -1118.6111, y: -520} + m_Pivot: {x: 0.5, y: 0.5} --- !u!1 &2144637746 GameObject: m_ObjectHideFlags: 0 @@ -981,4 +1038,4 @@ SceneRoots: - {fileID: 575233881} - {fileID: 1359769556} - {fileID: 2144637749} - - {fileID: 1969623180} + - {fileID: 709787123} diff --git a/Assets/Scripts/PoseCheck/EstimateModel/OpenPoseEsimater.cs b/Assets/Scripts/PoseCheck/EstimateModel/OpenPoseEsimater.cs index 06357bd..6c5f572 100644 --- a/Assets/Scripts/PoseCheck/EstimateModel/OpenPoseEsimater.cs +++ b/Assets/Scripts/PoseCheck/EstimateModel/OpenPoseEsimater.cs @@ -165,7 +165,7 @@ public class OpenPoseEsimater : Estimator if (!YogaManager.Instance.ActionCheckPoints(points)) { - LogPrint.Log("ActionCheckPoints failed. Re-Estimation."); + //LogPrint.Log("ActionCheckPoints failed. Re-Estimation."); return false; //重新检测 } return true; diff --git a/Assets/Scripts/PoseCheck/HoldPosition.cs b/Assets/Scripts/PoseCheck/HoldPosition.cs index f0a8dc6..8a7dceb 100644 --- a/Assets/Scripts/PoseCheck/HoldPosition.cs +++ b/Assets/Scripts/PoseCheck/HoldPosition.cs @@ -11,7 +11,7 @@ public class HoldPosition : PoseBase if (points.Count < 2) // 0级实现逻辑 不做操作 return null; - var distance = Vector2.zero; + float distance = 0; TimeSpan totalTimeSpan = TimeSpan.MinValue; var startPoint = points.FirstOrDefault().Item2; @@ -29,7 +29,7 @@ public class HoldPosition : PoseBase var p2 = points[i + 1].Item2; var vector = GetBasicVectorDirection(p1, p2); - distance += vector; + distance += vector.magnitude; TimeSpan timeSpan = points[i + 1].Item1 - points[i].Item1; totalTimeSpan = timeSpan; @@ -38,7 +38,7 @@ public class HoldPosition : PoseBase LogPrint.Warning($"angle: {angle}", PrintLevel.Normal); LogPrint.Log($"distance: {distance}, totalTimeSpan: {totalTimeSpan}", PrintLevel.Normal); - return (angle < 10 || MovementValidation(distance, totalTimeSpan, 1).GetValueOrDefault()); + return (angle < 8 && (distance/ points.Count < 8f)); } public override Vector2 GetBasicVectorDirection(List startPoint, List endPoint) @@ -66,7 +66,7 @@ public class HoldPosition : PoseBase //保持姿势不发生移动 - if (distance.magnitude < 15f)//允许一定误差 + if (distance.magnitude < 10f)//允许一定误差 { //EventManager.Instance.Dispatch(YogaEventType.Action_Success); return true; diff --git a/Assets/Scripts/Service/YogaManager.cs b/Assets/Scripts/Service/YogaManager.cs index 11dbe11..dc51b1b 100644 --- a/Assets/Scripts/Service/YogaManager.cs +++ b/Assets/Scripts/Service/YogaManager.cs @@ -9,6 +9,7 @@ using System.Collections.Generic; using System.Data; using System.Linq; using System.Runtime.CompilerServices; +using System.Xml.Serialization; using UnityEngine; using UnityEngine.Rendering.Universal; using UnityEngine.UI; @@ -112,6 +113,10 @@ public class YogaManager : MonoSingleton _currentCheckPointSuccessCount = 0; _comboTimes = 0; + _baseNose = -Vector2.one; + _baseLEye = -Vector2.one; + _baseREye = -Vector2.one; + //根据用户选择的动作索引，获取相应的资源 LevelData = YogaDataLoader.LoadData(LevelIndex); EventManager.Instance.Dispatch(YogaEventType.UpdateProgress, CurrentCheckPointSuccessCount, CurrentCheckPointCount); @@ -160,6 +165,7 @@ public class YogaManager : MonoSingleton } private void OnActionFailed() { + _comboTimes = 0; _currentCheckPointCount++; UpdateData(); } @@ -240,6 +246,8 @@ public class YogaManager : MonoSingleton #region 事件监听 + private AvatarAction _lastAction = AvatarAction.None; + private AvatarAction _currentAction = AvatarAction.None; private void OnSamplingStart() { @@ -258,13 +266,64 @@ public class YogaManager : MonoSingleton } try { + _lastAction = _currentAction; AvatarAction actionType = (AvatarAction)args.FirstOrDefault(); var startTime = ActionStartTime; _samplingResult = SampleQualityEvaluation(actionType, ActionStartTime, DateTime.Now); + _currentAction = actionType; //只有hold动作检测 if (actionType == AvatarAction.Hold) { _samplingYogaResult = _samplingResult; + var pointList = _estimateKeyPointsCache.Where(x => x.Item1 >= startTime && x.Item1 <= DateTime.Now).ToList(); + var nose = getAveragePointVector(pointList, "Nose"); + var neck = getAveragePointVector(pointList, "Neck"); + var LEye = getAveragePointVector(pointList, "LEye"); + var REye = getAveragePointVector(pointList, "REye"); + + if (_lastAction == AvatarAction.HeadTurnLeft) + { + if (_baseNose == -Vector2.one) + return; + var angle = Mathf.Abs(Vector2.SignedAngle(Vector2.left, (nose - _baseNose))); + _samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle < 90); + } + else if (_lastAction == AvatarAction.HeadTurnRight) + { + if (_baseNose == -Vector2.one) + return; + var angle = Mathf.Abs(Vector2.SignedAngle(Vector2.right, (nose - _baseNose))); + _samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle < 90); + } + else if (_lastAction == AvatarAction.HeadTurnDown) + { + if (_baseNose == -Vector2.one) + return; + var angle = Mathf.Abs(Vector2.SignedAngle(Vector2.down, -(nose - _baseNose)));//坐标系起始点左上，取y轴位移时需取反 + _samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle < 90); + } + else if (_lastAction == AvatarAction.HeadTurnUp) + { + if (_baseNose == -Vector2.one) + return; + var angle = Mathf.Abs(Vector2.SignedAngle(Vector2.up, -(nose - _baseNose))); //坐标系起始点左上，取y轴位移时需取反 + _samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle < 90); + } + else if (_lastAction == AvatarAction.LeftLateralHead) + { + if (_baseLEye == -Vector2.one || _baseREye == -Vector2.one) + return; + + var angle = Vector2.SignedAngle((LEye - REye), (_baseLEye - _baseREye)); + _samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle > 10); + } + else if (_lastAction == AvatarAction.RightLateralHead) + { + if (_baseLEye == -Vector2.one || _baseREye == -Vector2.one) + return; + var angle = Vector2.SignedAngle((LEye - REye), (_baseLEye - _baseREye)); + _samplingYogaResult = _samplingYogaResult.GetValueOrDefault() && (angle < -10); + } } } catch (Exception e) @@ -359,4 +418,43 @@ public class YogaManager : MonoSingleton return true; } + + private Vector2 _baseNose = -Vector2.one; + private Vector2 _baseNeck = -Vector2.one; + private Vector2 _baseLEye = -Vector2.one; + private Vector2 _baseREye = -Vector2.one; + + public void SetBasePosPoint() + { + //获取近1秒内所有关键点的平均值 + var cacheCopy = new List<(DateTime, List)>(_estimateKeyPointsCache); + var poinsList = cacheCopy.Where(x => x.Item1 >= DateTime.Now.AddSeconds(-1) && x.Item1 <= DateTime.Now).ToList(); + + _baseNose = getAveragePointVector(poinsList, "Nose"); + _baseNeck = getAveragePointVector(poinsList, "Neck"); + _baseLEye = getAveragePointVector(poinsList, "LEye"); + _baseREye = getAveragePointVector(poinsList, "REye"); + } + + private Vector2 getAveragePointVector(List<(DateTime, List)> poinsList, string partName) + { + int index = 0; + Vector2 retVal = -Vector2.one; + Vector2 totalVector = Vector2.zero; + foreach (var item in poinsList) + { + Vector2 vector = partName.vector(item.Item2); + if (vector != -Vector2.one) + { + totalVector += vector; + index++; + } + } + if (index != 0) + { + retVal = totalVector / index; + } + + return retVal; + } } \ No newline at end of file diff --git a/Assets/Scripts/Tool/PictureUtility.cs b/Assets/Scripts/Tool/PictureUtility.cs index ebbb744..8778d9a 100644 --- a/Assets/Scripts/Tool/PictureUtility.cs +++ b/Assets/Scripts/Tool/PictureUtility.cs @@ -1,5 +1,4 @@ - -using OpenCVForUnity.CoreModule; +using OpenCVCompact; using UnityEngine; public class PictureUtility @@ -30,27 +29,114 @@ public class PictureUtility return retVal; } - public static Mat HorizontalFlipMat(Mat mat) + public static OpenCVForUnity.CoreModule.Mat HorizontalFlipMat(OpenCVForUnity.CoreModule.Mat mat) { - Mat dst = new Mat(); - Core.flip(mat, dst, 1); + OpenCVForUnity.CoreModule.Mat dst = new OpenCVForUnity.CoreModule.Mat(); + OpenCVForUnity.CoreModule.Core.flip(mat, dst, 1); return dst; } //纠正鱼眼透视畸变 - public static Mat Undistort(Mat src) + public static OpenCVForUnity.CoreModule.Mat Undistort(OpenCVForUnity.CoreModule.Mat src) { //摄像机的4个畸变系数：k1,k2,k3,k4 - Mat K = new Mat(3, 3, CvType.CV_64FC1); + OpenCVForUnity.CoreModule.Mat K = new OpenCVForUnity.CoreModule.Mat(3, 3, OpenCVForUnity.CoreModule.CvType.CV_64FC1); K.put(0, 0, 1151.7, 0, 985.6, 0, 1153.4, 573.004, 0, 0, 1); - Mat D = new Mat(5, 1, CvType.CV_64FC1); + OpenCVForUnity.CoreModule.Mat D = new OpenCVForUnity.CoreModule.Mat(5, 1, OpenCVForUnity.CoreModule.CvType.CV_64FC1); //D.put(0, 0, -0.42, 0.2196, 0.0038, 0.013, -0.06); D.put(0, 0, -0.01, 0.09, 0, 0, -0.19); - Mat retVal = src.clone(); + OpenCVForUnity.CoreModule.Mat retVal = src.clone(); OpenCVForUnity.Calib3dModule.Calib3d.undistort(src, retVal, K, D); return retVal; } + + public static OpenCVCompact.Mat GetROIRect(OpenCVCompact.Mat bgrMat, out int[] roiRegion) + { + OpenCVCompact.Rect roiRect; + roiRegion = GetPositionRect(bgrMat.width(), bgrMat.height()); + roiRect = new OpenCVCompact.Rect( + (int)roiRegion[0], + (int)roiRegion[1], + Mathf.Abs((int)(roiRegion[2] - roiRegion[0])), + Mathf.Abs((int)(roiRegion[3] - roiRegion[1]))); + OpenCVCompact.Mat personRectImg = new OpenCVCompact.Mat(bgrMat, roiRect); + return personRectImg; + } + + public static int[] GetPositionRect(int width, int height) + { + int[] retVal = new int[4]; + int shift = (int)(width / 2 * 0.2f); + if (!GlobalData.Instance.IsDriverPosition) //副驾驶 + { + retVal[0] = 0; //left + retVal[1] = 0; //top + retVal[2] = (width / 2) - shift; //right + retVal[3] = height; //bottom + } + else + { + retVal[0] = (width / 2) + shift; //left + retVal[1] = 0; //top + retVal[2] = width; //right + retVal[3] = height; //bottom + } + return retVal; + } + + public static void DrawAimBox(OpenCVForUnity.CoreModule.Mat img, float[] box, float shift, int lineLength) + { + float left = box[0] + shift; + float top = box[1] + shift; + float right = box[2] - shift; + float bottom = box[3] - shift; + + OpenCVForUnity.ImgprocModule.Imgproc.cvtColor(img, img, OpenCVForUnity.ImgprocModule.Imgproc.COLOR_RGBA2RGB); + + //界面蓝 RGB = 19, 115, 230 + OpenCVForUnity.ImgprocModule.Imgproc.rectangle(img, new OpenCVForUnity.CoreModule.Point(left, top), new OpenCVForUnity.CoreModule.Point(right, bottom), new OpenCVForUnity.CoreModule.Scalar(255, 255, 255, 50), 2, 3); + //右下角 + OpenCVForUnity.ImgprocModule.Imgproc.line(img, new OpenCVForUnity.CoreModule.Point(right, bottom), new OpenCVForUnity.CoreModule.Point(right - lineLength, bottom), new OpenCVForUnity.CoreModule.Scalar(255, 255, 255, 255), 10, 3); + OpenCVForUnity.ImgprocModule.Imgproc.line(img, new OpenCVForUnity.CoreModule.Point(right, bottom), new OpenCVForUnity.CoreModule.Point(right, bottom - lineLength), new OpenCVForUnity.CoreModule.Scalar(255, 255, 255, 255), 10, 3); + //左下角 + OpenCVForUnity.ImgprocModule.Imgproc.line(img, new OpenCVForUnity.CoreModule.Point(left, bottom), new OpenCVForUnity.CoreModule.Point(left + lineLength, bottom), new OpenCVForUnity.CoreModule.Scalar(255, 255, 255, 255), 10, 3); + OpenCVForUnity.ImgprocModule.Imgproc.line(img, new OpenCVForUnity.CoreModule.Point(left, bottom), new OpenCVForUnity.CoreModule.Point(left, bottom - lineLength), new OpenCVForUnity.CoreModule.Scalar(255, 255, 255, 255), 10, 3); + //右上角 + OpenCVForUnity.ImgprocModule.Imgproc.line(img, new OpenCVForUnity.CoreModule.Point(right, top), new OpenCVForUnity.CoreModule.Point(right - lineLength, top), new OpenCVForUnity.CoreModule.Scalar(255, 255, 255, 255), 10, 3); + OpenCVForUnity.ImgprocModule.Imgproc.line(img, new OpenCVForUnity.CoreModule.Point(right, top), new OpenCVForUnity.CoreModule.Point(right, top + lineLength), new OpenCVForUnity.CoreModule.Scalar(255, 255, 255, 255), 10, 3); + //左上角 + OpenCVForUnity.ImgprocModule.Imgproc.line(img, new OpenCVForUnity.CoreModule.Point(left, top), new OpenCVForUnity.CoreModule.Point(left + lineLength, top), new OpenCVForUnity.CoreModule.Scalar(255, 255, 255, 255), 10, 3); + OpenCVForUnity.ImgprocModule.Imgproc.line(img, new OpenCVForUnity.CoreModule.Point(left, top), new OpenCVForUnity.CoreModule.Point(left, top + lineLength), new OpenCVForUnity.CoreModule.Scalar(255, 255, 255, 255), 10, 3); + OpenCVForUnity.ImgprocModule.Imgproc.cvtColor(img, img, OpenCVForUnity.ImgprocModule.Imgproc.COLOR_RGB2RGBA); + } + + public static void DrawAimBoxFaceDetect(OpenCVCompact.Mat img, float[] box, float shift, int lineLength) + { + float left = box[0] + shift; + float top = box[1] + shift; + float right = box[2] - shift; + float bottom = box[3] - shift; + + OpenCVCompact.Imgproc.cvtColor(img, img, OpenCVCompact.Imgproc.COLOR_RGBA2RGB); + + //界面蓝 RGB = 19, 115, 230 + OpenCVCompact.Imgproc.rectangle(img, new OpenCVCompact.Point(left, top), new OpenCVCompact.Point(right, bottom), new OpenCVCompact.Scalar(255, 255, 255, 50), 2, 3); + //右下角 + OpenCVCompact.Imgproc.line(img, new OpenCVCompact.Point(right, bottom), new OpenCVCompact.Point(right - lineLength, bottom), new OpenCVCompact.Scalar(255, 255, 255, 255), 10, 3); + OpenCVCompact.Imgproc.line(img, new OpenCVCompact.Point(right, bottom), new OpenCVCompact.Point(right, bottom - lineLength), new OpenCVCompact.Scalar(255, 255, 255, 255), 10, 3); + //左下角 + OpenCVCompact.Imgproc.line(img, new OpenCVCompact.Point(left, bottom), new OpenCVCompact.Point(left + lineLength, bottom), new OpenCVCompact.Scalar(255, 255, 255, 255), 10, 3); + OpenCVCompact.Imgproc.line(img, new OpenCVCompact.Point(left, bottom), new OpenCVCompact.Point(left, bottom - lineLength), new OpenCVCompact.Scalar(255, 255, 255, 255), 10, 3); + //右上角 + OpenCVCompact.Imgproc.line(img, new OpenCVCompact.Point(right, top), new OpenCVCompact.Point(right - lineLength, top), new OpenCVCompact.Scalar(255, 255, 255, 255), 10, 3); + OpenCVCompact.Imgproc.line(img, new OpenCVCompact.Point(right, top), new OpenCVCompact.Point(right, top + lineLength), new OpenCVCompact.Scalar(255, 255, 255, 255), 10, 3); + //左上角 + OpenCVCompact.Imgproc.line(img, new OpenCVCompact.Point(left, top), new OpenCVCompact.Point(left + lineLength, top), new OpenCVCompact.Scalar(255, 255, 255, 255), 10, 3); + OpenCVCompact.Imgproc.line(img, new OpenCVCompact.Point(left, top), new OpenCVCompact.Point(left, top + lineLength), new OpenCVCompact.Scalar(255, 255, 255, 255), 10, 3); + OpenCVCompact.Imgproc.cvtColor(img, img, OpenCVCompact.Imgproc.COLOR_RGB2RGBA); + } } + diff --git a/Assets/Scripts/UI/Component/CaptureManagerBase.cs b/Assets/Scripts/UI/Component/CaptureManagerBase.cs index ad2d434..6f629bc 100644 --- a/Assets/Scripts/UI/Component/CaptureManagerBase.cs +++ b/Assets/Scripts/UI/Component/CaptureManagerBase.cs @@ -47,7 +47,7 @@ namespace Yoga MatInitial(); CVEstimator.Instance.Init();//姿态检测模型初始化 } - + private void OnEnable() { EventRegist(); @@ -81,14 +81,8 @@ namespace Yoga if (YogaManager.Instance.PersonRectResult.Count > 0) { - var box = YogaManager.Instance.PersonRectResult[0]; - float left = box[0]; - float top = box[1]; - float right = box[2]; - float bottom = box[3]; //Imgproc.rectangle(img, new Point(left, top), new Point(right, bottom), new Scalar(0, 0, 255), 5, 3); - - DrawAimBox(img, left + 10, top + 10, right - 10, bottom - 10, 40); + PictureUtility.DrawAimBox(img, YogaManager.Instance.PersonRectResult[0], 10, 40); } //打印 @@ -125,26 +119,7 @@ namespace Yoga Utils.matToTexture2D(img, DisplayTexture); } - private static void DrawAimBox(Mat img, float left, float top, float right, float bottom, int lineLength) - { - Imgproc.cvtColor(img, img, Imgproc.COLOR_RGBA2RGB); - //界面蓝 RGB = 19, 115, 230 - Imgproc.rectangle(img, new Point(left, top), new Point(right, bottom), new Scalar(255, 255, 255, 50), 2, 3); - //右下角 - Imgproc.line(img, new Point(right, bottom), new Point(right - lineLength, bottom), new Scalar(255, 255, 255, 255), 10, 3); - Imgproc.line(img, new Point(right, bottom), new Point(right, bottom - lineLength), new Scalar(255, 255, 255, 255), 10, 3); - //左下角 - Imgproc.line(img, new Point(left, bottom), new Point(left + lineLength, bottom), new Scalar(255, 255, 255, 255), 10, 3); - Imgproc.line(img, new Point(left, bottom), new Point(left, bottom - lineLength), new Scalar(255, 255, 255, 255), 10, 3); - //右上角 - Imgproc.line(img, new Point(right, top), new Point(right - lineLength, top), new Scalar(255, 255, 255, 255), 10, 3); - Imgproc.line(img, new Point(right, top), new Point(right, top + lineLength), new Scalar(255, 255, 255, 255), 10, 3); - //左上角 - Imgproc.line(img, new Point(left, top), new Point(left + lineLength, top), new Scalar(255, 255, 255, 255), 10, 3); - Imgproc.line(img, new Point(left, top), new Point(left, top + lineLength), new Scalar(255, 255, 255, 255), 10, 3); - Imgproc.cvtColor(img, img, Imgproc.COLOR_RGB2RGBA); - } private void OnDestroy() { diff --git a/Assets/Scripts/UI/Component/FaceDetectManager.cs b/Assets/Scripts/UI/Component/FaceDetectManagerBase.cs similarity index 72% rename from Assets/Scripts/UI/Component/FaceDetectManager.cs rename to Assets/Scripts/UI/Component/FaceDetectManagerBase.cs index e2d8125..12b0b19 100644 --- a/Assets/Scripts/UI/Component/FaceDetectManager.cs +++ b/Assets/Scripts/UI/Component/FaceDetectManagerBase.cs @@ -2,28 +2,25 @@ using OpenCVCompact; using System; using UnityEngine; -using OpenCVForUnity.UnityUtils.Helper; using System.Collections.Generic; +using OpenCVForUnity.UnityUtils.Helper; +using dnn; -[RequireComponent(typeof(OpenCVCompact.WebCamTextureToMatHelper))] -public class FaceDetectManager : MonoBehaviour +public abstract class FaceDetectManagerBase : MonoBehaviour { //Related To DNN Model - dnn.DNNUtils dnnUtils; + protected dnn.DNNUtils dnnUtils; - //Image Capture - OpenCVCompact.WebCamTextureToMatHelper _webCamTextureToMatHelper; - Texture2D videoTexture; - private double _lndmrkEstScore; - private Mat mat4Process; - private Mat grayMat4Process; - private Mat mat4Display; - private Mat mat4DisplayTexture; - private Mat lndmrk; - private MatOfRect detectionResult; - private Rect detectRect; - int[] faceRect = new int[4]; - bool webCamReady = false; + protected Texture2D videoTexture; + protected double _lndmrkEstScore; + protected Mat mat4Process; + protected Mat grayMat4Process; + protected Mat mat4Display; + protected Mat mat4DisplayTexture; + protected Mat lndmrk; + protected MatOfRect detectionResult; + protected Rect detectRect; + protected int[] faceRect = new int[4]; //int webCamOrVideoOrImage = 0; //0 : webcam, 1 : video, 2 : image int lndmrkMode = 1; //0: 51, 1: 84 @@ -32,7 +29,7 @@ public class FaceDetectManager : MonoBehaviour float lndmrkEstScore; - private FaceDetectUI panel; + protected FaceDetectUI panel; // Start is called before the first frame update void Awake() @@ -42,9 +39,7 @@ public class FaceDetectManager : MonoBehaviour InitModel(); InitVariable(); - - _webCamTextureToMatHelper = gameObject.GetComponent(); - _webCamTextureToMatHelper.Initialize(); + InitMatHelper(); } private void OnDestroy() @@ -73,8 +68,10 @@ public class FaceDetectManager : MonoBehaviour //if (probExp != null) // probExp.Dispose(); - _webCamTextureToMatHelper.Dispose(); } + protected abstract void InitMatHelper(); + + protected abstract void DestroyManager(); private void InitVariable() { @@ -176,29 +173,31 @@ public class FaceDetectManager : MonoBehaviour return true; } - private void AnalyzingFace() - { - if (!_webCamTextureToMatHelper.IsPlaying() || !_webCamTextureToMatHelper.DidUpdateThisFrame()) - return; + protected abstract Mat GetMat(); - mat4Display = _webCamTextureToMatHelper.GetMat(); + //update + protected virtual void AnalyzingFace() + { + mat4Display = GetMat(); + if (mat4Display == null) + return; //mat4Display.copyTo(mat4Process); - mat4Process = mat4Display.clone(); + + + mat4Process = PictureUtility.GetROIRect(mat4Display.clone(), out var roiRegion); //Face Detect panel.RefreshData(LandmarkDetect()); - if (webCamReady == true) - { - if (mat4Display.rows() == videoTexture.height) - { - mat4Display.copyTo(mat4DisplayTexture); - Utils.matToTexture2D(mat4DisplayTexture, videoTexture); - } - } + var point1 = new Point(roiRegion[0], roiRegion[1]); + var point2 = new Point(roiRegion[0] + roiRegion[2], roiRegion[0] + roiRegion[3]); + Imgproc.rectangle(mat4Display, point1, point2, new Scalar(255, 0, 0), 2); + DebugPint(); } - private Dictionary LandmarkDetect() + protected abstract void DebugPint(); + + protected Dictionary LandmarkDetect() { if (dnnUtils.GetEstimateLandmarkSuccessOrNot() == false || lndmrkEstScore < 0.25) //If It Failed To Track Facial Landmark in Previous Frame { @@ -242,55 +241,9 @@ public class FaceDetectManager : MonoBehaviour idx = idx + 1; } - LogPrint.Warning(text.Substring(0, text.Length - 1)); } prob.Dispose(); return retVal; } - - public void OnWebCamTextureToMatHelperInitialized() - { - Mat webCamTextureMat = _webCamTextureToMatHelper.GetMat(); - - videoTexture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.RGBA32, false); - gameObject.GetComponent().material.mainTexture = videoTexture; - gameObject.transform.localScale = new Vector3(webCamTextureMat.cols(), webCamTextureMat.rows(), 1); - - mat4Display = new Mat(webCamTextureMat.rows(), webCamTextureMat.cols(), CvType.CV_8UC4); - mat4DisplayTexture = new Mat(webCamTextureMat.rows(), webCamTextureMat.cols(), CvType.CV_8UC4); - - LogPrint.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); - - float width = webCamTextureMat.width(); - float height = webCamTextureMat.height(); - - float widthScale = (float)Screen.width / width; - float heightScale = (float)Screen.height / height; - if (widthScale < heightScale) - { - Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2; - } - else - { - Camera.main.orthographicSize = height / 2; - } - - dnnUtils.InitHeadPoseEstimationCameraInfo(webCamTextureMat.cols(), webCamTextureMat.rows()); - - webCamReady = true; - LogPrint.Log("OnWebCamTextureToMatHelperInitialized"); - } - - - public void OnWebCamTextureToMatHelperDisposed() - { - LogPrint.Log("OnWebCamTextureToMatHelperDisposed"); - - } - - public void OnWebCamTextureToMatHelperErrorOccurred(OpenCVCompact.WebCamTextureToMatHelper.ErrorCode errorCode) - { - LogPrint.Log("OnWebCamTextureToMatHelperErrorOccurred " + errorCode); - } } diff --git a/Assets/Scripts/UI/Component/FaceDetectManager.cs.meta b/Assets/Scripts/UI/Component/FaceDetectManagerBase.cs.meta similarity index 100% rename from Assets/Scripts/UI/Component/FaceDetectManager.cs.meta rename to Assets/Scripts/UI/Component/FaceDetectManagerBase.cs.meta diff --git a/Assets/Scripts/UI/Component/GudieAnimationManager.cs b/Assets/Scripts/UI/Component/GudieAnimationManager.cs index 4958e9c..579e43b 100644 --- a/Assets/Scripts/UI/Component/GudieAnimationManager.cs +++ b/Assets/Scripts/UI/Component/GudieAnimationManager.cs @@ -12,23 +12,22 @@ public class GudieAnimationManager : MonoBehaviour private List _actionNameList = new List(); - private float _animationLength = 0; public float AnimationLength { get { - if (_animationLength == 0) + float length = 0; + AnimationClip[] clips = Animator.runtimeAnimatorController.animationClips; + foreach (AnimationClip clip in clips) { - AnimationClip[] clips = Animator.runtimeAnimatorController.animationClips; - foreach (AnimationClip clip in clips) + if (ActionNameList.Contains(clip.name)) { - if (ActionNameList.Contains(clip.name)) - { - _animationLength += clip.length; - } + length += clip.length; + LogPrint.Log("clip.name:" + clip.name + " clip.length:" + clip.length); + LogPrint.Log("length:" + length); } } - return _animationLength; + return length; } } @@ -48,9 +47,10 @@ public class GudieAnimationManager : MonoBehaviour { get { - if (_actionNameList.Count == 0) + if (_actionNameList.Count != YogaManager.Instance.LevelData.Actions.Count) { - foreach (var item in _actionList) + _actionNameList.Clear(); + foreach (var item in YogaManager.Instance.LevelData.Actions) { _actionNameList.Add(item.ToString()); } @@ -77,17 +77,6 @@ public class GudieAnimationManager : MonoBehaviour { _actionList = actionList; - AnimationClip[] clips = Animator.runtimeAnimatorController.animationClips; - _animationLength = 0; - foreach (AnimationClip clip in clips) - { - if (_actionNameList.Contains(clip.name)) - { - _animationLength += clip.length; - } - } - - Play(Enum.GetName(typeof(AvatarAction), _actionList[YogaManager.Instance.ActionIndex])); } @@ -97,7 +86,7 @@ public class GudieAnimationManager : MonoBehaviour return; //当未达到指标且动画播放完毕时，重新播放 - if (Animator.GetCurrentAnimatorStateInfo(0).normalizedTime >= 1.0f) + if (Animator.GetCurrentAnimatorStateInfo(0).normalizedTime >= 1.0f && GetCurrentAnimationName().Equals(Enum.GetName(typeof(AvatarAction), _actionList[YogaManager.Instance.ActionIndex]))) { //如果列表动画index小于列表长度，播放下一个动画 YogaManager.Instance.ActionIndex++; diff --git a/Assets/Scripts/UI/Component/USBFaceDetectManager.cs b/Assets/Scripts/UI/Component/USBFaceDetectManager.cs new file mode 100644 index 0000000..70a7e98 --- /dev/null +++ b/Assets/Scripts/UI/Component/USBFaceDetectManager.cs @@ -0,0 +1,196 @@ +using OpenCVCompact; +using Serenegiant.UVC; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using UnityEngine; +using UnityEngine.UI; + +public class USBFaceDetectManager : FaceDetectManagerBase, IUVCDrawer +{ + public RawImage image; + public RawImage cutImage; + private Material _flipImg; + private int[] _positionRect; + private Texture2D _cutTexture; + protected override void InitMatHelper() + { + _flipImg = new Material(Shader.Find("Unlit/FlipHorizontal")); ; + InvokeRepeating("GCCollection", 0, 1f); + } + + protected void GCCollection() + { + if (enabled == true && gameObject.activeInHierarchy == true) + StartCoroutine(CleanUp()); + } + protected IEnumerator CleanUp() + { + GC.Collect(); + yield return Resources.UnloadUnusedAssets(); + GC.Collect(); + } + + protected override Mat GetMat() + { + Mat img = null; + if (UVCManager.Instance != null && + UVCManager.Instance.GetAttachedDevices() != null && UVCManager.Instance.GetAttachedDevices().Count > 0) + { + var devices = UVCManager.Instance.GetAttachedDevices(); + for (int i = 0; i < devices.Count; i++) + { + UVCManager.CameraInfo device = devices[i]; + //LogPrint.Log($"device: {device}, device name:{device.DeviceName} texture: {device.previewTexture}"); + } + //image.texture = devices.FirstOrDefault().previewTexture; + textureToFlipTexture2D(devices.FirstOrDefault().previewTexture, videoTexture); + + img = new Mat(UVCManager.Instance.DefaultHeight, UVCManager.Instance.DefaultWidth, CvType.CV_8UC4); + Utils.texture2DToMat(videoTexture, img); + } + + return img; + } + private void textureToFlipTexture2D(Texture texture, Texture2D texture2D) + { + if (texture == null) + throw new ArgumentNullException("texture"); + + if (texture2D == null) + throw new ArgumentNullException("texture2D"); + + if (texture.width != texture2D.width || texture.height != texture2D.height) + throw new ArgumentException("texture and texture2D need to be the same size."); + + RenderTexture prevRT = RenderTexture.active; + + RenderTexture tempRT = RenderTexture.GetTemporary(texture.width, texture.height, 0, RenderTextureFormat.ARGB32); + Graphics.Blit(texture, tempRT, _flipImg); + + RenderTexture.active = tempRT; + texture2D.ReadPixels(new UnityEngine.Rect(0f, 0f, texture.width, texture.height), 0, 0, false); + texture2D.Apply(false, false); + RenderTexture.ReleaseTemporary(tempRT); + + RenderTexture.active = prevRT; + } + + //protected override void AnalyzingFace() + //{ + // mat4Display = GetMat(); + // if (mat4Display == null) + // return; + // //mat4Display.copyTo(mat4Process); + // var roiRect = new OpenCVCompact.Rect( + // _positionRect[0], + // _positionRect[1], + // Mathf.Abs(_positionRect[2] - _positionRect[0]), + // Mathf.Abs(_positionRect[3] - _positionRect[1])); + + // mat4Process = new Mat(mat4Display, roiRect); + + + // //Face Detect + // panel.RefreshData(LandmarkDetect()); + + + + // Imgproc.rectangle(mat4Display, roiRect, new Scalar(255, 0, 0), 2); + // DebugPint(); + //} + + protected override void DebugPint() + { + //image.texture = videoTexture; + if (mat4Process.rows() == _cutTexture.height) + { + LogPrint.Log("mat4Process.rows() " + mat4Process.rows() + " mat4Process.cols() " + mat4Process.cols()); + Mat cutMat = mat4Process.clone(); + + LogPrint.Log($"mat4Display type:{mat4Display.type()}.{CvType.CV_8UC3},{CvType.CV_8UC4}"); + + //Utils.matToTexture2D(cutMat, _cutTexture); + } + + if (mat4Display.rows() == videoTexture.height) + { + //mat4Display.copyTo(mat4DisplayTexture); + Mat display = mat4Display.clone(); + //Utils.matToTexture2D(display, videoTexture); + } + } + + protected override void DestroyManager() + { + } + + #region UVC + public UVCFilter[] UVCFilters; + private Texture SavedTexture; + public bool OnUVCAttachEvent(UVCManager manager, UVCDevice device) + { + var result = !device.IsRicoh || device.IsTHETA; + + result &= UVCFilter.Match(device, UVCFilters); + + return result; + } + + public void OnUVCDetachEvent(UVCManager manager, UVCDevice device) + { + LogPrint.Log("OnUVCDetachEvent " + device); + } + + public bool CanDraw(UVCManager manager, UVCDevice device) + { + return UVCFilter.Match(device, UVCFilters); + } + + public void OnUVCStartEvent(UVCManager manager, UVCDevice device, Texture tex) + { + HandleOnStartPreview(tex); + } + + public void OnUVCStopEvent(UVCManager manager, UVCDevice device) + { + HandleOnStopPreview(); + } + + private void HandleOnStartPreview(Texture tex) + { + SavedTexture = videoTexture; + + videoTexture = new Texture2D(UVCManager.Instance.DefaultWidth, UVCManager.Instance.DefaultHeight, TextureFormat.RGBA32, false); + _positionRect = PictureUtility.GetPositionRect(UVCManager.Instance.DefaultWidth, UVCManager.Instance.DefaultHeight); + _cutTexture = new Texture2D(_positionRect[2], _positionRect[3], TextureFormat.RGBA32, false); + image.texture = videoTexture; + cutImage.texture = _cutTexture; + + //mat4Display = new Mat(UVCManager.Instance.DefaultWidth, UVCManager.Instance.DefaultHeight, CvType.CV_8UC4); + mat4DisplayTexture = new Mat(UVCManager.Instance.DefaultWidth, UVCManager.Instance.DefaultHeight, CvType.CV_8UC4); + + LogPrint.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); + + dnnUtils.InitHeadPoseEstimationCameraInfo(UVCManager.Instance.DefaultWidth, UVCManager.Instance.DefaultHeight); + } + private void HandleOnStopPreview() + { + RestoreTexture(); + } + + private void RestoreTexture() + { + try + { + transform.GetComponent().material.mainTexture = SavedTexture; + } + catch (Exception e) + { + Debug.LogException(e); + } + SavedTexture = null; + } + #endregion +} diff --git a/Assets/Scripts/UI/Component/USBFaceDetectManager.cs.meta b/Assets/Scripts/UI/Component/USBFaceDetectManager.cs.meta new file mode 100644 index 0000000..2e0e759 --- /dev/null +++ b/Assets/Scripts/UI/Component/USBFaceDetectManager.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: b1a3af86bea67944b9cce9661ba1bfc6 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Scripts/UI/Component/WebCamFaceDetectManager.cs b/Assets/Scripts/UI/Component/WebCamFaceDetectManager.cs new file mode 100644 index 0000000..f0a3bb6 --- /dev/null +++ b/Assets/Scripts/UI/Component/WebCamFaceDetectManager.cs @@ -0,0 +1,99 @@ +using OpenCVCompact; +using Serenegiant.UVC; +using UnityEngine; +using UnityEngine.UI; + +[RequireComponent(typeof(OpenCVCompact.WebCamTextureToMatHelper))] +public class WebCamFaceDetectManager : FaceDetectManagerBase +{ + public RawImage image; + public RawImage cutImage; + bool webCamReady = false; + private int[] _positionRect; + private Texture2D _cutTexture; + + //Image Capture + OpenCVCompact.WebCamTextureToMatHelper _webCamTextureToMatHelper; + protected override void InitMatHelper() + { + _webCamTextureToMatHelper = gameObject.GetComponent(); + _webCamTextureToMatHelper.Initialize(); + } + protected override Mat GetMat() + { + if (!_webCamTextureToMatHelper.IsPlaying() || !_webCamTextureToMatHelper.DidUpdateThisFrame()) + return null; + return _webCamTextureToMatHelper.GetMat(); + } + + protected override void DestroyManager() + { + _webCamTextureToMatHelper.Dispose(); + } + protected override void DebugPint() + { + if (webCamReady == true) + { + if (mat4Display.rows() == videoTexture.height) + { + mat4Display.copyTo(mat4DisplayTexture); + Utils.matToTexture2D(mat4DisplayTexture, videoTexture); + + } + if (mat4Process.rows() == cutImage.texture.height) + { + Mat cutMat = mat4Process.clone(); + Texture2D cutTexture = new Texture2D(cutMat.cols(), cutMat.rows(), TextureFormat.RGBA32, false); + Utils.matToTexture2D(cutMat, cutTexture); + cutImage.texture = cutTexture; + } + } + } + public void OnWebCamTextureToMatHelperInitialized() + { + Mat webCamTextureMat = _webCamTextureToMatHelper.GetMat(); + + videoTexture = new Texture2D(webCamTextureMat.cols(), webCamTextureMat.rows(), TextureFormat.RGBA32, false); + //gameObject.GetComponent().material.mainTexture = videoTexture; + image.texture = videoTexture; + _positionRect = PictureUtility.GetPositionRect(webCamTextureMat.cols(), webCamTextureMat.rows()); + _cutTexture = new Texture2D(_positionRect[2], _positionRect[3], TextureFormat.RGBA32, false); + cutImage.texture = _cutTexture; + + mat4Display = new Mat(webCamTextureMat.rows(), webCamTextureMat.cols(), CvType.CV_8UC4); + mat4DisplayTexture = new Mat(webCamTextureMat.rows(), webCamTextureMat.cols(), CvType.CV_8UC4); + + LogPrint.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation); + + float width = webCamTextureMat.width(); + float height = webCamTextureMat.height(); + + float widthScale = (float)Screen.width / width; + float heightScale = (float)Screen.height / height; + if (widthScale < heightScale) + { + Camera.main.orthographicSize = (width * (float)Screen.height / (float)Screen.width) / 2; + } + else + { + Camera.main.orthographicSize = height / 2; + } + + dnnUtils.InitHeadPoseEstimationCameraInfo(webCamTextureMat.cols(), webCamTextureMat.rows()); + + webCamReady = true; + LogPrint.Log("OnWebCamTextureToMatHelperInitialized"); + } + + + public void OnWebCamTextureToMatHelperDisposed() + { + LogPrint.Log("OnWebCamTextureToMatHelperDisposed"); + + } + + public void OnWebCamTextureToMatHelperErrorOccurred(OpenCVCompact.WebCamTextureToMatHelper.ErrorCode errorCode) + { + LogPrint.Log("OnWebCamTextureToMatHelperErrorOccurred " + errorCode); + } +} \ No newline at end of file diff --git a/Assets/Scripts/UI/Component/WebCamFaceDetectManager.cs.meta b/Assets/Scripts/UI/Component/WebCamFaceDetectManager.cs.meta new file mode 100644 index 0000000..93d085d --- /dev/null +++ b/Assets/Scripts/UI/Component/WebCamFaceDetectManager.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 23c90b7e25f07ce4298ce424fedeb5e4 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Scripts/UI/FaceDetectUI.cs b/Assets/Scripts/UI/FaceDetectUI.cs index f9eaf2f..76f1419 100644 --- a/Assets/Scripts/UI/FaceDetectUI.cs +++ b/Assets/Scripts/UI/FaceDetectUI.cs @@ -8,7 +8,7 @@ public class FaceDetectUI : UIPanelBase private Dictionary itemDetail = new Dictionary(); private Transform _content; - private List _lockedItem = new List() { "Blond_Hair", "Wearing_Hat" }; + private List _lockedItem = new List() { "Blond_Hair", "Wearing_Hat" }; private void Awake() { @@ -40,7 +40,7 @@ public class FaceDetectUI : UIPanelBase { if (item.Key.Equals("Bald")) continue; - + GameObject go; if (index < count) { @@ -72,8 +72,22 @@ public class FaceDetectUI : UIPanelBase index++; } + + AddFixObj("Heart rate"); //Heart rate + AddFixObj("Blood oxygen"); //Blood oxygen } + private void AddFixObj(string title) + { + var fixGo = Instantiate(ItemPrefab, _content); + var mgr = fixGo.GetComponent(); + mgr.Title.text = title; + fixGo.transform.gameObject.SetActive(true); + fixGo.transform.SetSiblingIndex(_content.transform.childCount); + mgr.Lock(); + } + + public void OnBackBtnClicked() { UIManager.Instance.CloseCurrent(); diff --git a/Assets/Scripts/UI/GuideUI.cs b/Assets/Scripts/UI/GuideUI.cs index 8c808a9..de1e20b 100644 --- a/Assets/Scripts/UI/GuideUI.cs +++ b/Assets/Scripts/UI/GuideUI.cs @@ -183,7 +183,7 @@ public class GuideUI : UIPanelBase YogaManager.Instance.ActionIndex = 0; EventManager.Instance.Dispatch(YogaEventType.PlayAnimation, YogaManager.Instance.LevelData.Actions); _startTime = DateTime.Now; - InvokeRepeating("ProgressUpdate", 0, 0.05f); //20fps + InvokeRepeating("ProgressUpdate", 0, 0.1f); //20fps } public override void OnExit() diff --git a/Assets/Scripts/UI/RobotController.cs b/Assets/Scripts/UI/RobotController.cs index f26f875..fbc723d 100644 --- a/Assets/Scripts/UI/RobotController.cs +++ b/Assets/Scripts/UI/RobotController.cs @@ -7,6 +7,11 @@ using static UnityEngine.Rendering.GPUSort; public class RobotController : MonoBehaviour { + public void SetBasePosPoint() + { + YogaManager.Instance.SetBasePosPoint(); + } + public void ActionEnd() { AudioManager.Instance.PlayCVInQueue("End"); diff --git a/Assets/UI/icon.png b/Assets/UI/icon.png new file mode 100644 index 0000000..676c8a1 Binary files /dev/null and b/Assets/UI/icon.png differ diff --git a/Assets/UI/icon.png.meta b/Assets/UI/icon.png.meta new file mode 100644 index 0000000..de8404d --- /dev/null +++ b/Assets/UI/icon.png.meta @@ -0,0 +1,124 @@ +fileFormatVersion: 2 +guid: 22f3692ba25f06c4cbb69c64cd7ef9f7 +TextureImporter: + internalIDToNameTable: [] + externalObjects: {} + serializedVersion: 12 + mipmaps: + mipMapMode: 0 + enableMipMap: 1 + sRGBTexture: 1 + linearTexture: 0 + fadeOut: 0 + borderMipMap: 0 + mipMapsPreserveCoverage: 0 + alphaTestReferenceValue: 0.5 + mipMapFadeDistanceStart: 1 + mipMapFadeDistanceEnd: 3 + bumpmap: + convertToNormalMap: 0 + externalNormalMap: 0 + heightScale: 0.25 + normalMapFilter: 0 + flipGreenChannel: 0 + isReadable: 0 + streamingMipmaps: 0 + streamingMipmapsPriority: 0 + vTOnly: 0 + ignoreMipmapLimit: 0 + grayScaleToAlpha: 0 + generateCubemap: 6 + cubemapConvolution: 0 + seamlessCubemap: 0 + textureFormat: 1 + maxTextureSize: 2048 + textureSettings: + serializedVersion: 2 + filterMode: 1 + aniso: 1 + mipBias: 0 + wrapU: 0 + wrapV: 0 + wrapW: 0 + nPOTScale: 1 + lightmap: 0 + compressionQuality: 50 + spriteMode: 0 + spriteExtrude: 1 + spriteMeshType: 1 + alignment: 0 + spritePivot: {x: 0.5, y: 0.5} + spritePixelsToUnits: 100 + spriteBorder: {x: 0, y: 0, z: 0, w: 0} + spriteGenerateFallbackPhysicsShape: 1 + alphaUsage: 1 + alphaIsTransparency: 0 + spriteTessellationDetail: -1 + textureType: 0 + textureShape: 1 + singleChannelComponent: 0 + flipbookRows: 1 + flipbookColumns: 1 + maxTextureSizeSet: 0 + compressionQualitySet: 0 + textureFormatSet: 0 + ignorePngGamma: 0 + applyGammaDecoding: 0 + swizzle: 50462976 + cookieLightType: 0 + platformSettings: + - serializedVersion: 3 + buildTarget: DefaultTexturePlatform + maxTextureSize: 2048 + resizeAlgorithm: 0 + textureFormat: -1 + textureCompression: 1 + compressionQuality: 50 + crunchedCompression: 0 + allowsAlphaSplitting: 0 + overridden: 0 + androidETC2FallbackOverride: 0 + forceMaximumCompressionQuality_BC6H_BC7: 0 + - serializedVersion: 3 + buildTarget: Standalone + maxTextureSize: 2048 + resizeAlgorithm: 0 + textureFormat: -1 + textureCompression: 1 + compressionQuality: 50 + crunchedCompression: 0 + allowsAlphaSplitting: 0 + overridden: 0 + androidETC2FallbackOverride: 0 + forceMaximumCompressionQuality_BC6H_BC7: 0 + - serializedVersion: 3 + buildTarget: Android + maxTextureSize: 2048 + resizeAlgorithm: 0 + textureFormat: -1 + textureCompression: 1 + compressionQuality: 50 + crunchedCompression: 0 + allowsAlphaSplitting: 0 + overridden: 0 + androidETC2FallbackOverride: 0 + forceMaximumCompressionQuality_BC6H_BC7: 0 + spriteSheet: + serializedVersion: 2 + sprites: [] + outline: [] + physicsShape: [] + bones: [] + spriteID: + internalID: 0 + vertices: [] + indices: + edges: [] + weights: [] + secondaryTextures: [] + nameFileIdTable: {} + mipmapLimitGroupName: + pSDRemoveMatte: 0 + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/UVC4UnityAndroidPlugin/Scripts/UVCManager.cs b/Assets/UVC4UnityAndroidPlugin/Scripts/UVCManager.cs index f63bede..bb6a9f2 100644 --- a/Assets/UVC4UnityAndroidPlugin/Scripts/UVCManager.cs +++ b/Assets/UVC4UnityAndroidPlugin/Scripts/UVCManager.cs @@ -1,4 +1,4 @@ -#define ENABLE_LOG +//#define ENABLE_LOG /* * Copyright (c) 2014 - 2022 t_saki@serenegiant.com */ @@ -715,11 +715,15 @@ namespace Serenegiant.UVC Debug.Log($"{TAG}InitPlugin:has no IUVCDrawer, try to get from gameObject"); #endif var objs = FindObjectsByType(FindObjectsSortMode.None); + var faceObjs = FindObjectsByType(FindObjectsSortMode.None); List drawers = new List(); foreach (var obj in objs) { drawers.AddRange(obj.GetComponents(typeof(IUVCDrawer))); - + } + foreach (var obj in faceObjs) + { + drawers.AddRange(obj.GetComponents(typeof(IUVCDrawer))); } LogPrint.Warning("drawers: " + drawers.Count); if ((drawers != null) && (drawers.Count > 0))