From c5186f75b567303e6b58777d0ddea2aaba20e6e5 Mon Sep 17 00:00:00 2001 From: terric Date: Wed, 15 Nov 2023 16:10:56 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8B=86=E5=88=86=E6=A8=A1=E5=9E=8B=E5=8A=A0?= =?UTF-8?q?=E8=BD=BD+UI=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Assets/Resources/UI/ActionListUI.prefab | 274 +++++++++++++++++- Assets/Resources/UI/Background.prefab | 76 ----- Assets/Resources/UI/HUDRoot.prefab | 42 ++- Assets/Scenes/YogaMain.unity | 7 +- Assets/Scripts/Boot.cs | 12 +- Assets/Scripts/CVEstimator.cs | 214 -------------- Assets/Scripts/PoseCheck/EstimateModel.meta | 8 + .../PoseCheck/EstimateModel/CVEstimator.cs | 144 +++++++++ .../EstimateModel}/CVEstimator.cs.meta | 0 .../PoseCheck/EstimateModel/Estimator.cs | 134 +++++++++ .../PoseCheck/EstimateModel/Estimator.cs.meta | 11 + .../EstimateModel/MediaPipeEstimator.cs | 21 ++ .../EstimateModel/MediaPipeEstimator.cs.meta | 11 + .../EstimateModel/OpenPoseEsimater.cs | 89 ++++++ .../EstimateModel/OpenPoseEsimater.cs.meta | 11 + .../Scripts/PoseCheck/MotionCaptureManager.cs | 113 ++------ Assets/Scripts/UI/ActionListUIPanel.cs | 7 + Assets/Scripts/UI/LoadingManager.cs | 6 +- Assets/Scripts/UI/UIManager.cs | 11 + Assets/Scripts/YogaConfig.cs | 7 +- Assets/Scripts/YogaDataLoader.cs | 13 + Assets/Scripts/YogaManager.cs | 13 +- Assets/UI/Img/Arrow Simple Left.png | Bin 0 -> 6163 bytes Assets/UI/Img/Arrow Simple Left.png.meta | 124 ++++++++ 24 files changed, 942 insertions(+), 406 deletions(-) delete mode 100644 Assets/Scripts/CVEstimator.cs create mode 100644 Assets/Scripts/PoseCheck/EstimateModel.meta create mode 100644 Assets/Scripts/PoseCheck/EstimateModel/CVEstimator.cs rename Assets/Scripts/{ => PoseCheck/EstimateModel}/CVEstimator.cs.meta (100%) create mode 100644 Assets/Scripts/PoseCheck/EstimateModel/Estimator.cs create mode 100644 Assets/Scripts/PoseCheck/EstimateModel/Estimator.cs.meta create mode 100644 Assets/Scripts/PoseCheck/EstimateModel/MediaPipeEstimator.cs create mode 100644 Assets/Scripts/PoseCheck/EstimateModel/MediaPipeEstimator.cs.meta create mode 100644 Assets/Scripts/PoseCheck/EstimateModel/OpenPoseEsimater.cs create mode 100644 Assets/Scripts/PoseCheck/EstimateModel/OpenPoseEsimater.cs.meta create mode 100644 Assets/UI/Img/Arrow Simple Left.png create mode 100644 Assets/UI/Img/Arrow Simple Left.png.meta diff --git a/Assets/Resources/UI/ActionListUI.prefab b/Assets/Resources/UI/ActionListUI.prefab index 84b7312..fb68fd8 100644 --- a/Assets/Resources/UI/ActionListUI.prefab +++ b/Assets/Resources/UI/ActionListUI.prefab @@ -950,7 +950,7 @@ MonoBehaviour: m_Arguments: m_ObjectArgument: {fileID: 0} m_ObjectArgumentAssemblyTypeName: UnityEngine.Object, UnityEngine - m_IntArgument: 5 + m_IntArgument: 4 m_FloatArgument: 0 m_StringArgument: m_BoolArgument: 0 @@ -3499,7 +3499,7 @@ MonoBehaviour: m_Arguments: m_ObjectArgument: {fileID: 0} m_ObjectArgumentAssemblyTypeName: UnityEngine.Object, UnityEngine - m_IntArgument: 3 + m_IntArgument: 2 m_FloatArgument: 0 m_StringArgument: m_BoolArgument: 0 @@ -5707,6 +5707,140 @@ MonoBehaviour: keepAlphaValue: 1 useCustomColor: 0 colorType: 2 +--- !u!1 &3587488935766229191 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 8025476141745189055} + - component: {fileID: 5565143401225979560} + - component: {fileID: 5503859348676013585} + m_Layer: 5 + m_Name: Text (TMP) + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &8025476141745189055 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 3587488935766229191} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 1, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: [] + m_Father: {fileID: 1975068002469358035} + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0, y: 0} + m_AnchorMax: {x: 1, y: 1} + m_AnchoredPosition: {x: 0, y: 0} + m_SizeDelta: {x: 0, y: 0} + m_Pivot: {x: 0.5, y: 0.5} +--- !u!222 &5565143401225979560 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 3587488935766229191} + m_CullTransparentMesh: 1 +--- !u!114 &5503859348676013585 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 3587488935766229191} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: f4688fdb7df04437aeb418b961361dc5, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 1, g: 1, b: 1, a: 1} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_text: + m_isRightToLeft: 0 + m_fontAsset: {fileID: 11400000, guid: 8f586378b4e144a9851e7b34d9b748ee, type: 2} + m_sharedMaterial: {fileID: 2180264, guid: 8f586378b4e144a9851e7b34d9b748ee, type: 2} + m_fontSharedMaterials: [] + m_fontMaterial: {fileID: 0} + m_fontMaterials: [] + m_fontColor32: + serializedVersion: 2 + rgba: 4281479730 + m_fontColor: {r: 0.19607843, g: 0.19607843, b: 0.19607843, a: 1} + m_enableVertexGradient: 0 + m_colorMode: 3 + m_fontColorGradient: + topLeft: {r: 1, g: 1, b: 1, a: 1} + topRight: {r: 1, g: 1, b: 1, a: 1} + bottomLeft: {r: 1, g: 1, b: 1, a: 1} + bottomRight: {r: 1, g: 1, b: 1, a: 1} + m_fontColorGradientPreset: {fileID: 0} + m_spriteAsset: {fileID: 0} + m_tintAllSprites: 0 + m_StyleSheet: {fileID: 0} + m_TextStyleHashCode: -1183493901 + m_overrideHtmlColors: 0 + m_faceColor: + serializedVersion: 2 + rgba: 4294967295 + m_fontSize: 24 + m_fontSizeBase: 24 + m_fontWeight: 400 + m_enableAutoSizing: 0 + m_fontSizeMin: 18 + m_fontSizeMax: 72 + m_fontStyle: 0 + m_HorizontalAlignment: 2 + m_VerticalAlignment: 512 + m_textAlignment: 65535 + m_characterSpacing: 0 + m_wordSpacing: 0 + m_lineSpacing: 0 + m_lineSpacingMax: 0 + m_paragraphSpacing: 0 + m_charWidthMaxAdj: 0 + m_enableWordWrapping: 1 + m_wordWrappingRatios: 0.4 + m_overflowMode: 0 + m_linkedTextComponent: {fileID: 0} + parentLinkedComponent: {fileID: 0} + m_enableKerning: 1 + m_enableExtraPadding: 0 + checkPaddingRequired: 0 + m_isRichText: 1 + m_parseCtrlCharacters: 1 + m_isOrthographic: 1 + m_isCullingEnabled: 0 + m_horizontalMapping: 0 + m_verticalMapping: 0 + m_uvLineOffset: 0 + m_geometrySortingOrder: 0 + m_IsTextObjectScaleStatic: 0 + m_VertexBufferAutoSizeReduction: 0 + m_useMaxVisibleDescender: 1 + m_pageToDisplay: 1 + m_margin: {x: 0, y: 0, z: 0, w: 0} + m_isUsingLegacyAnimationComponent: 0 + m_isVolumetricText: 0 + m_hasFontAssetChanged: 0 + m_baseMaterial: {fileID: 0} + m_maskOffset: {x: 0, y: 0, z: 0, w: 0} --- !u!1 &3635298159763109128 GameObject: m_ObjectHideFlags: 0 @@ -6324,6 +6458,7 @@ RectTransform: m_Children: - {fileID: 2445187332351384993} - {fileID: 458346039371182555} + - {fileID: 1975068002469358035} m_Father: {fileID: 6472390402513143558} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} @@ -7580,7 +7715,7 @@ MonoBehaviour: m_Arguments: m_ObjectArgument: {fileID: 0} m_ObjectArgumentAssemblyTypeName: UnityEngine.Object, UnityEngine - m_IntArgument: 4 + m_IntArgument: 3 m_FloatArgument: 0 m_StringArgument: m_BoolArgument: 0 @@ -10844,6 +10979,139 @@ CanvasRenderer: m_PrefabAsset: {fileID: 0} m_GameObject: {fileID: 8229600306201579037} m_CullTransparentMesh: 0 +--- !u!1 &8265266211400670364 +GameObject: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + serializedVersion: 6 + m_Component: + - component: {fileID: 1975068002469358035} + - component: {fileID: 1474344257591453156} + - component: {fileID: 3105870774428680706} + - component: {fileID: 4592116318987236448} + m_Layer: 5 + m_Name: back + m_TagString: Untagged + m_Icon: {fileID: 0} + m_NavMeshLayer: 0 + m_StaticEditorFlags: 0 + m_IsActive: 1 +--- !u!224 &1975068002469358035 +RectTransform: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 8265266211400670364} + m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} + m_LocalPosition: {x: 0, y: 0, z: 0} + m_LocalScale: {x: 0.5, y: 1, z: 1} + m_ConstrainProportionsScale: 0 + m_Children: + - {fileID: 8025476141745189055} + m_Father: {fileID: 3228235001121865301} + m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} + m_AnchorMin: {x: 0.5, y: 0.5} + m_AnchorMax: {x: 0.5, y: 0.5} + m_AnchoredPosition: {x: -832, y: 0} + m_SizeDelta: {x: 256, y: 256} + m_Pivot: {x: 0.5, y: 0.5} +--- !u!222 &1474344257591453156 +CanvasRenderer: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 8265266211400670364} + m_CullTransparentMesh: 1 +--- !u!114 &3105870774428680706 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 8265266211400670364} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Material: {fileID: 0} + m_Color: {r: 0.34509805, g: 0.34509805, b: 0.34509805, a: 1} + m_RaycastTarget: 1 + m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} + m_Maskable: 1 + m_OnCullStateChanged: + m_PersistentCalls: + m_Calls: [] + m_Sprite: {fileID: 21300000, guid: 96b53986fbf913d47864b19cbe98e372, type: 3} + m_Type: 0 + m_PreserveAspect: 0 + m_FillCenter: 1 + m_FillMethod: 4 + m_FillAmount: 1 + m_FillClockwise: 1 + m_FillOrigin: 0 + m_UseSpriteMesh: 0 + m_PixelsPerUnitMultiplier: 1 +--- !u!114 &4592116318987236448 +MonoBehaviour: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 8265266211400670364} + m_Enabled: 1 + m_EditorHideFlags: 0 + m_Script: {fileID: 11500000, guid: 4e29b1a8efbd4b44bb3f3716e73f07ff, type: 3} + m_Name: + m_EditorClassIdentifier: + m_Navigation: + m_Mode: 3 + m_WrapAround: 0 + m_SelectOnUp: {fileID: 0} + m_SelectOnDown: {fileID: 0} + m_SelectOnLeft: {fileID: 0} + m_SelectOnRight: {fileID: 0} + m_Transition: 1 + m_Colors: + m_NormalColor: {r: 1, g: 1, b: 1, a: 1} + m_HighlightedColor: {r: 0.9607843, g: 0.9607843, b: 0.9607843, a: 1} + m_PressedColor: {r: 0.78431374, g: 0.78431374, b: 0.78431374, a: 1} + m_SelectedColor: {r: 0.9607843, g: 0.9607843, b: 0.9607843, a: 1} + m_DisabledColor: {r: 0.78431374, g: 0.78431374, b: 0.78431374, a: 0.5019608} + m_ColorMultiplier: 1 + m_FadeDuration: 0.1 + m_SpriteState: + m_HighlightedSprite: {fileID: 0} + m_PressedSprite: {fileID: 0} + m_SelectedSprite: {fileID: 0} + m_DisabledSprite: {fileID: 0} + m_AnimationTriggers: + m_NormalTrigger: Normal + m_HighlightedTrigger: Highlighted + m_PressedTrigger: Pressed + m_SelectedTrigger: Selected + m_DisabledTrigger: Disabled + m_Interactable: 1 + m_TargetGraphic: {fileID: 3105870774428680706} + m_OnClick: + m_PersistentCalls: + m_Calls: + - m_Target: {fileID: -5447994580174514295} + m_TargetAssemblyTypeName: ActionListUIPanel, Assembly-CSharp + m_MethodName: OnBackBtnClicked + m_Mode: 1 + m_Arguments: + m_ObjectArgument: {fileID: 0} + m_ObjectArgumentAssemblyTypeName: UnityEngine.Object, UnityEngine + m_IntArgument: 0 + m_FloatArgument: 0 + m_StringArgument: + m_BoolArgument: 0 + m_CallState: 2 --- !u!1 &8275927407155805902 GameObject: m_ObjectHideFlags: 0 diff --git a/Assets/Resources/UI/Background.prefab b/Assets/Resources/UI/Background.prefab index d1cded4..5896915 100644 --- a/Assets/Resources/UI/Background.prefab +++ b/Assets/Resources/UI/Background.prefab @@ -88,81 +88,6 @@ CanvasGroup: m_Interactable: 1 m_BlocksRaycasts: 1 m_IgnoreParentGroups: 0 ---- !u!1 &1381910008050731583 -GameObject: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - serializedVersion: 6 - m_Component: - - component: {fileID: 2790109640146298654} - - component: {fileID: 2214680058021086237} - - component: {fileID: 40271058087427464} - m_Layer: 5 - m_Name: Logo - m_TagString: Untagged - m_Icon: {fileID: 0} - m_NavMeshLayer: 0 - m_StaticEditorFlags: 0 - m_IsActive: 1 ---- !u!224 &2790109640146298654 -RectTransform: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 1381910008050731583} - m_LocalRotation: {x: 0, y: 0, z: 0, w: 1} - m_LocalPosition: {x: 0, y: 0, z: 0} - m_LocalScale: {x: 1, y: 1, z: 1} - m_ConstrainProportionsScale: 0 - m_Children: [] - m_Father: {fileID: 8805156796672989800} - m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} - m_AnchorMin: {x: 0, y: 1} - m_AnchorMax: {x: 0, y: 1} - m_AnchoredPosition: {x: 20, y: -20} - m_SizeDelta: {x: 200, y: 100} - m_Pivot: {x: 0, y: 1} ---- !u!222 &2214680058021086237 -CanvasRenderer: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 1381910008050731583} - m_CullTransparentMesh: 1 ---- !u!114 &40271058087427464 -MonoBehaviour: - m_ObjectHideFlags: 0 - m_CorrespondingSourceObject: {fileID: 0} - m_PrefabInstance: {fileID: 0} - m_PrefabAsset: {fileID: 0} - m_GameObject: {fileID: 1381910008050731583} - m_Enabled: 1 - m_EditorHideFlags: 0 - m_Script: {fileID: 11500000, guid: fe87c0e1cc204ed48ad3b37840f39efc, type: 3} - m_Name: - m_EditorClassIdentifier: - m_Material: {fileID: 0} - m_Color: {r: 1, g: 1, b: 1, a: 1} - m_RaycastTarget: 1 - m_RaycastPadding: {x: 0, y: 0, z: 0, w: 0} - m_Maskable: 1 - m_OnCullStateChanged: - m_PersistentCalls: - m_Calls: [] - m_Sprite: {fileID: 21300000, guid: eb6f879a267ac444d86df8427a559c97, type: 3} - m_Type: 0 - m_PreserveAspect: 0 - m_FillCenter: 1 - m_FillMethod: 4 - m_FillAmount: 1 - m_FillClockwise: 1 - m_FillOrigin: 0 - m_UseSpriteMesh: 0 - m_PixelsPerUnitMultiplier: 1 --- !u!1 &4327728351019041244 GameObject: m_ObjectHideFlags: 0 @@ -194,7 +119,6 @@ RectTransform: m_Children: - {fileID: 5390399916831871626} - {fileID: 5279216800401342767} - - {fileID: 2790109640146298654} m_Father: {fileID: 0} m_LocalEulerAnglesHint: {x: 0, y: 0, z: 0} m_AnchorMin: {x: 0, y: 0} diff --git a/Assets/Resources/UI/HUDRoot.prefab b/Assets/Resources/UI/HUDRoot.prefab index 7b723cd..ae6d70c 100644 --- a/Assets/Resources/UI/HUDRoot.prefab +++ b/Assets/Resources/UI/HUDRoot.prefab @@ -10,6 +10,8 @@ GameObject: m_Component: - component: {fileID: 7882980980823895129} - component: {fileID: 44101575073041383} + - component: {fileID: 4721913218893670612} + - component: {fileID: 262072879874942703} m_Layer: 5 m_Name: HUDRoot m_TagString: Untagged @@ -49,6 +51,40 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: 685773266ab8a444e9ff5131cde3c8ae, type: 3} m_Name: m_EditorClassIdentifier: +--- !u!95 &4721913218893670612 +Animator: + serializedVersion: 7 + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 591297386902869668} + m_Enabled: 1 + m_Avatar: {fileID: 0} + m_Controller: {fileID: 0} + m_CullingMode: 0 + m_UpdateMode: 0 + m_ApplyRootMotion: 0 + m_LinearVelocityBlending: 0 + m_StabilizeFeet: 0 + m_AnimatePhysics: 0 + m_WarningMessage: + m_HasTransformHierarchy: 1 + m_AllowConstantClipSamplingOptimization: 1 + m_KeepAnimatorStateOnDisable: 0 + m_WriteDefaultValuesOnDisable: 0 +--- !u!225 &262072879874942703 +CanvasGroup: + m_ObjectHideFlags: 0 + m_CorrespondingSourceObject: {fileID: 0} + m_PrefabInstance: {fileID: 0} + m_PrefabAsset: {fileID: 0} + m_GameObject: {fileID: 591297386902869668} + m_Enabled: 1 + m_Alpha: 1 + m_Interactable: 1 + m_BlocksRaycasts: 1 + m_IgnoreParentGroups: 0 --- !u!1001 &1332753061905480905 PrefabInstance: m_ObjectHideFlags: 0 @@ -91,7 +127,7 @@ PrefabInstance: objectReference: {fileID: 0} - target: {fileID: 224602918499771590, guid: a5c1511e3af132c44b6f11542f709726, type: 3} propertyPath: m_AnchorMax.x - value: 0 + value: 0.5 objectReference: {fileID: 0} - target: {fileID: 224602918499771590, guid: a5c1511e3af132c44b6f11542f709726, type: 3} propertyPath: m_AnchorMax.y @@ -99,7 +135,7 @@ PrefabInstance: objectReference: {fileID: 0} - target: {fileID: 224602918499771590, guid: a5c1511e3af132c44b6f11542f709726, type: 3} propertyPath: m_AnchorMin.x - value: 0 + value: 0.5 objectReference: {fileID: 0} - target: {fileID: 224602918499771590, guid: a5c1511e3af132c44b6f11542f709726, type: 3} propertyPath: m_AnchorMin.y @@ -143,7 +179,7 @@ PrefabInstance: objectReference: {fileID: 0} - target: {fileID: 224602918499771590, guid: a5c1511e3af132c44b6f11542f709726, type: 3} propertyPath: m_AnchoredPosition.x - value: 230 + value: 0 objectReference: {fileID: 0} - target: {fileID: 224602918499771590, guid: a5c1511e3af132c44b6f11542f709726, type: 3} propertyPath: m_AnchoredPosition.y diff --git a/Assets/Scenes/YogaMain.unity b/Assets/Scenes/YogaMain.unity index 07fdbdc..6e87737 100644 --- a/Assets/Scenes/YogaMain.unity +++ b/Assets/Scenes/YogaMain.unity @@ -937,7 +937,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: df35b0c19ca97734e87299a664cea35f, type: 3} m_Name: m_EditorClassIdentifier: - _requestedDeviceName: 1 + _requestedDeviceName: 2 _requestedWidth: 1280 _requestedHeight: 960 _requestedIsFrontFacing: 0 @@ -1005,11 +1005,6 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: c96807d839ba7554d92494fd6d25bb28, type: 3} m_Name: m_EditorClassIdentifier: - inpWidth: 416 - inpHeight: 416 - confThreshold: 0.35 - nmsThreshold: 0.6 - topK: 1000 SamplingRate: 0.1 --- !u!1 &1680658391 GameObject: diff --git a/Assets/Scripts/Boot.cs b/Assets/Scripts/Boot.cs index dfdb589..3f2dd5b 100644 --- a/Assets/Scripts/Boot.cs +++ b/Assets/Scripts/Boot.cs @@ -3,6 +3,7 @@ using UnityEngine.Rendering; public class Boot : MonoBehaviour { + private bool _isInited = false; [RuntimeInitializeOnLoadMethod(RuntimeInitializeLoadType.BeforeSplashScreen)] public static void StopLogo() { @@ -19,13 +20,20 @@ public class Boot : MonoBehaviour Screen.sleepTimeout = SleepTimeout.NeverSleep; //Ļ Screen.orientation = ScreenOrientation.LandscapeLeft; + _isInited = UIManager.IsInited; } private void OnEnable() { // - UIManager.Instance.Init(); - UIManager.Instance.ShowPanel(); + //UIManager.Instance.Init(); + if (!_isInited) + UIManager.Instance.ShowPanel(); + else + { + UIManager.Instance.LoadReset(); + UIManager.Instance.ShowPanel(); + } } // Update is called once per frame diff --git a/Assets/Scripts/CVEstimator.cs b/Assets/Scripts/CVEstimator.cs deleted file mode 100644 index bcef135..0000000 --- a/Assets/Scripts/CVEstimator.cs +++ /dev/null @@ -1,214 +0,0 @@ -using OpenCVForUnity.CoreModule; -using OpenCVForUnity.DnnModule; -using OpenCVForUnity.ImgprocModule; -using OpenCVForUnity.UnityUtils; -using OpenCVForUnity.UnityUtils.Helper; -using System; -using System.Collections.Generic; -using System.Threading; -using UnityEngine; -using Yoga; - -public class CVEstimator : Singleton -{ - private bool isInited = false; - - public bool IsInited { get => isInited; private set => isInited = value; } - public bool IsRunning { get; private set; } - - private Thread _mainThread; - private Thread _estimateThread; - - private static readonly TimeSpan _refreshRate = TimeSpan.FromSeconds(1 / 30); - - public void Init(params object[] args) - { - if (IsInited) - { - return; - } - if (_mainThread != null) - { - _mainThread.Abort(); - _mainThread = null; - } - - //先用YOLOv7检测人体，再用OpenPose检测人体姿态 - if (args.Length < 2) - throw new ArgumentException("CVEstimator.Init: args.Length < 2"); - - _objectDetector = args[0] as YOLOv7ObjectDetector; - if (_objectDetector == null) - throw new ArgumentException("CVEstimator.Init: args[0] is not YOLOv7ObjectDetector"); - - _openPoseModel = args[1] as KeypointsModel; - if (_openPoseModel == null) - throw new ArgumentException("CVEstimator.Init: args[1] is not KeypointsModel"); - - _mainThread = new Thread(new ThreadStart(MainThread)); - _mainThread.Start(); - - IsInited = true; - } - - public void StartEstimation() - { - if (!IsInited) - { - Debug.LogError("CVEstimator is not inited. "); - return; - } - _estimateThread.Start(); - if (IsRunning) - { - return; - } - IsRunning = true; - } - - public void Dispose() - { - if (!IsInited) - { - return; - } - IsRunning = false; - _estimateThread.Abort(); - _mainThread.Abort(); - IsInited = false; - } - - private YOLOv7ObjectDetector _objectDetector; - public int inpWidth = 416; - public int inpHeight = 416; - public float confThreshold = 0.35f; - public float nmsThreshold = 0.6f; - public int topK = 1000; - private double threshold = 0.5; - - private Net _net; - private KeypointsModel _openPoseModel; - - private void MainThread() - { - //加载模型 - Utils.setDebugMode(true); //打印日志 - //解析数据 - _estimateThread = new Thread(new ThreadStart(Estimation)); - } - - private void Estimation() - { - DateTime startTime = DateTime.Now; - Mat bgrMat = new Mat(); - while (IsRunning) - { - if (_objectDetector == null) - { - Debug.LogWarning("ObjectDetector is not ready. "); - IsRunning = false; //停止检测 - } - - //获取数据 - Mat rgbaMat = MotionCaptureManager.Instance.RgbaMat; - if (rgbaMat == null) - { - Debug.Log("WebCamTexture is null. "); - Debug.Log("Re-Estimation."); - continue; //重新检测 - } - - Imgproc.cvtColor(rgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR); - Mat results = _objectDetector.infer(bgrMat); - var voloResultBox = new List(); - bool hasValidObject = false; - for (int i = results.rows() - 1; i >= 0; --i) - { - float[] box = new float[4]; - results.get(i, 0, box); //方框 - float[] conf = new float[1]; - results.get(i, 4, conf); //检测数据 - float[] cls = new float[1]; - results.get(i, 5, cls); //类别 - - if (!IsObjectValid(box, conf, cls, rgbaMat)) - { - continue; - } - hasValidObject = true; - voloResultBox.Clear(); - voloResultBox.Add(box); - voloResultBox.Add(conf); - voloResultBox.Add(cls); - break; - } - - if (!hasValidObject) //没有检测到人体 - { - Debug.Log("No person block found. Re-Estimation."); - continue; //重新检测 - } - //更新人物框检测结果 - MotionCaptureManager.Instance.VoloResult = voloResultBox; - - OpenCVForUnity.CoreModule.Rect roiRect = new OpenCVForUnity.CoreModule.Rect( - (int)voloResultBox[0][0], - (int)voloResultBox[0][1], - Math.Abs((int)(voloResultBox[0][2] - voloResultBox[0][0])), - Math.Abs((int)(voloResultBox[0][3] - voloResultBox[0][1]))); - - if (roiRect.y < 0 || //0 <= _rowRange.start - (roiRect.y + roiRect.height) < roiRect.y || // _rowRange.start <= _rowRange.end - bgrMat.rows() < (roiRect.y + roiRect.height)) //_rowRange.end <= m.rows - continue; //重新检测 - - Mat personRectImg = new Mat(bgrMat, roiRect);//获取人体区域 - List points = _openPoseModel.estimate(personRectImg, (float)threshold).toList(); - //将人体区域的坐标转换为原图坐标 - for (int j = 0; j < points.Count; j++) - { - if (points[j] == null || - (points[j].x == -1 && points[j].y == -1)) //没找到的点，跳过 - continue; - - points[j].x += roiRect.x; - points[j].y += roiRect.y; - } - - if (!YogaManager.Instance.ActionCheckPoints(points)) - { - Debug.Log("ActionCheckPoints failed. Re-Estimation."); - continue; //重新检测 - } - - //更新关键点位检测结果 - MotionCaptureManager.Instance.CurrPersonPoints = points; - - //等待到下一个更新周期 - if (DateTime.Now - startTime > _refreshRate) - { - startTime = DateTime.Now; - } - else - { - Thread.Sleep(_refreshRate - (DateTime.Now - startTime)); - } - } - bgrMat.Dispose(); - } - - private bool IsObjectValid(float[] box, float[] confidence, float[] classID, Mat rgbaMat) - { - if ((int)classID[0] != 0 || confidence[0] < 0.8f) //只检测人体，且置信度大于80% - return false; - - //是否满足主驾/副驾的位置条件 - float width = rgbaMat.width(); - //获取矩形的中点 - float centerX = (box[0] + box[2]) / 2; - //左边副驾驶，右边主驾驶 - - - return true; - } -} \ No newline at end of file diff --git a/Assets/Scripts/PoseCheck/EstimateModel.meta b/Assets/Scripts/PoseCheck/EstimateModel.meta new file mode 100644 index 0000000..75ea353 --- /dev/null +++ b/Assets/Scripts/PoseCheck/EstimateModel.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 5dfab41ece4615444a5c6384f28728eb +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Scripts/PoseCheck/EstimateModel/CVEstimator.cs b/Assets/Scripts/PoseCheck/EstimateModel/CVEstimator.cs new file mode 100644 index 0000000..f568761 --- /dev/null +++ b/Assets/Scripts/PoseCheck/EstimateModel/CVEstimator.cs @@ -0,0 +1,144 @@ +using OpenCVForUnity.CoreModule; +using OpenCVForUnity.DnnModule; +using OpenCVForUnity.ImgprocModule; +using OpenCVForUnity.UnityUtils; +using OpenCVForUnity.UnityUtils.Helper; +using System; +using System.Collections.Generic; +using System.Threading; +using UnityEngine; +using Yoga; + +public class CVEstimator : Singleton +{ + private bool isInited = false; + + public bool IsInited { get => isInited; private set => isInited = value; } + public bool IsRunning { get; private set; } + + private Thread _mainThread; + private Thread _estimateThread; + + private static readonly TimeSpan _refreshRate = TimeSpan.FromSeconds(1 / 30); + + //public int inpWidth = 416; + //public int inpHeight = 416; + //public float confThreshold = 0.35f; + //public float nmsThreshold = 0.6f; + //public int topK = 1000; + //private double threshold = 0.5; + + private Net _net; + private KeypointsModel _openPoseModel; + + public void Init(params object[] args) + { + if (IsInited) + { + return; + } + if (_mainThread != null) + { + _mainThread.Abort(); + _mainThread = null; + } + + //先用YOLOv7检测人体，再用OpenPose检测人体姿态 + //if (args.Length < 1) + // throw new ArgumentException("CVEstimator.Init: args.Length < 2"); + + //_objectDetector = args[0] as YOLOv7ObjectDetector; + //if (_objectDetector == null) + // throw new ArgumentException("CVEstimator.Init: args[0] is not YOLOv7ObjectDetector"); + + //_openPoseModel = args[1] as KeypointsModel; + //if (_openPoseModel == null) + // throw new ArgumentException("CVEstimator.Init: args[1] is not KeypointsModel"); + + _mainThread = new Thread(new ThreadStart(MainThread)); + _mainThread.Start(); + + IsInited = true; + } + + public void StartEstimation() + { + if (!IsInited) + { + Debug.LogError("CVEstimator is not inited. "); + return; + } + _estimateThread.Start(); + if (IsRunning) + { + return; + } + IsRunning = true; + } + + public void Dispose() + { + if (!IsInited) + { + return; + } + IsRunning = false; + _estimateThread.Abort(); + _mainThread.Abort(); + IsInited = false; + } + + private void MainThread() + { + //加载模型 + Utils.setDebugMode(true); //打印日志 + //解析数据 + _estimateThread = new Thread(new ThreadStart(Estimation)); + } + + private void Estimation() + { + DateTime startTime = DateTime.Now; + Mat bgrMat = new Mat(); + + while (IsRunning) + { + Mat rgbaMat = MotionCaptureManager.Instance.RgbaMat; + if (rgbaMat == null) + { + Debug.Log("WebCamTexture is null. "); + Debug.Log("Re-Estimation."); + continue; //重新检测 + } + Imgproc.cvtColor(rgbaMat, bgrMat, Imgproc.COLOR_RGBA2BGR); + + if (!YogaManager.Instance.CurrentEstimator.TryGetROIRegionMat(rgbaMat, bgrMat, out List voloResultBox)) + { + continue; //重新检测 + } + + //更新人物框检测结果 + MotionCaptureManager.Instance.VoloResult = voloResultBox; + + if (!YogaManager.Instance.CurrentEstimator.Esitmate(bgrMat, voloResultBox, out List points)) + { + Debug.LogWarning("Pose estimation failed. Re-Estimating."); + continue; //重新检测 + } + + //更新关键点位检测结果 + MotionCaptureManager.Instance.CurrPersonPoints = points; + + //等待到下一个更新周期 + if (DateTime.Now - startTime > _refreshRate) + { + startTime = DateTime.Now; + } + else + { + Thread.Sleep(_refreshRate - (DateTime.Now - startTime)); + } + } + bgrMat.Dispose(); + } +} \ No newline at end of file diff --git a/Assets/Scripts/CVEstimator.cs.meta b/Assets/Scripts/PoseCheck/EstimateModel/CVEstimator.cs.meta similarity index 100% rename from Assets/Scripts/CVEstimator.cs.meta rename to Assets/Scripts/PoseCheck/EstimateModel/CVEstimator.cs.meta diff --git a/Assets/Scripts/PoseCheck/EstimateModel/Estimator.cs b/Assets/Scripts/PoseCheck/EstimateModel/Estimator.cs new file mode 100644 index 0000000..33fd708 --- /dev/null +++ b/Assets/Scripts/PoseCheck/EstimateModel/Estimator.cs @@ -0,0 +1,134 @@ +using OpenCVForUnity.CoreModule; +using OpenCVForUnity.ImgprocModule; +using OpenCVForUnity.UnityUtils; +using OpenCVForUnity.UnityUtils.Helper; +using System; +using System.Collections.Generic; +using UnityEngine; +using Yoga; + +public abstract class Estimator +{ + private YOLOv7ObjectDetector _objectDetector; + public int inpWidth = 416; + public int inpHeight = 416; + public float confThreshold = 0.35f; + public float nmsThreshold = 0.6f; + public int topK = 1000; + + //初始化 + public void Init() + { + _objectDetector = new YOLOv7ObjectDetector( + Utils.getFilePath("OpenCVForUnity/dnn/yolov7-tiny.weights"), + Utils.getFilePath("OpenCVForUnity/dnn/yolov7-tiny.cfg"), + Utils.getFilePath("OpenCVForUnity/dnn/coco.names"), + new Size(inpWidth, inpHeight), confThreshold, nmsThreshold/*, topK*/); + + InitModel(); + } + + public abstract void InitModel(); + + public void Dispose() + { + DisposeModel(); + } + public abstract void DisposeModel(); + + //检测到的人体框 默认使用VOLO7 + public virtual bool TryGetROIRegionMat(Mat rgbaMat, Mat bgrMat, out List voloResult) + { + voloResult = null; + if (_objectDetector == null) + { + Debug.LogWarning("ObjectDetector is not ready. "); + return false; //重新检测 + } + + //获取数据 + Mat results = _objectDetector.infer(bgrMat); + var voloResultBox = new List(); + bool hasValidObject = false; + for (int i = results.rows() - 1; i >= 0; --i) + { + float[] box = new float[4]; + results.get(i, 0, box); //方框 + float[] conf = new float[1]; + results.get(i, 4, conf); //检测数据 + float[] cls = new float[1]; + results.get(i, 5, cls); //类别 + + if (!IsObjectValid(box, conf, cls, rgbaMat)) + { + return false; + } + hasValidObject = true; + voloResultBox.Clear(); + voloResultBox.Add(box); + voloResultBox.Add(conf); + voloResultBox.Add(cls); + break; + } + + if (!hasValidObject) //没有检测到人体 + { + Debug.Log("No person block found. Re-Estimation."); + return false; //重新检测 + } + + voloResult = voloResultBox; + return true; + } + + + protected bool IsObjectValid(float[] box, float[] confidence, float[] classID, Mat rgbaMat) + { + if ((int)classID[0] != 0 || confidence[0] < 0.8f) //只检测人体，且置信度大于80% + return false; + + //是否满足主驾/副驾的位置条件 + float width = rgbaMat.width(); + //获取矩形的中点 + float centerX = (box[0] + box[2]) / 2; + //左边副驾驶，右边主驾驶 + + return true; + } + + public void DebugPrintObjectLayout(Mat image, float[] box, float[] conf, float[] cls, bool isRGB = false) + { + + float left = box[0]; + float top = box[1]; + float right = box[2]; + float bottom = box[3]; + int classId = (int)cls[0]; + + Scalar c = _objectDetector.palette[classId % _objectDetector.palette.Count]; + Scalar color = isRGB ? c : new Scalar(c.val[2], c.val[1], c.val[0], c.val[3]); + + Imgproc.rectangle(image, new Point(left, top), new Point(right, bottom), color, 2); + + string label = String.Format("{0:0.00}", conf[0]); + if (_objectDetector.classNames != null && _objectDetector.classNames.Count != 0) + { + if (classId < (int)_objectDetector.classNames.Count) + { + label = _objectDetector.classNames[classId] + " " + label; + } + } + + int[] baseLine = new int[1]; + Size labelSize = Imgproc.getTextSize(label, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine); + + top = Mathf.Max((float)top, (float)labelSize.height); + Imgproc.rectangle(image, new Point(left, top - labelSize.height), + new Point(left + labelSize.width, top + baseLine[0]), color, Core.FILLED); + Imgproc.putText(image, label, new Point(left, top), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, Scalar.all(255), 1, Imgproc.LINE_AA); + } + + public abstract bool Esitmate(Mat bgrMat, List voloResultBox, out List points); + + public virtual void Check(ref Mat img) { } +} diff --git a/Assets/Scripts/PoseCheck/EstimateModel/Estimator.cs.meta b/Assets/Scripts/PoseCheck/EstimateModel/Estimator.cs.meta new file mode 100644 index 0000000..17a8587 --- /dev/null +++ b/Assets/Scripts/PoseCheck/EstimateModel/Estimator.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 9b182376ae5163a4e8f57f947c81ddc1 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Scripts/PoseCheck/EstimateModel/MediaPipeEstimator.cs b/Assets/Scripts/PoseCheck/EstimateModel/MediaPipeEstimator.cs new file mode 100644 index 0000000..a13491b --- /dev/null +++ b/Assets/Scripts/PoseCheck/EstimateModel/MediaPipeEstimator.cs @@ -0,0 +1,21 @@ +using OpenCVForUnity.CoreModule; +using System; +using System.Collections.Generic; + +public class MediaPipeEstimator : Estimator +{ + public override void InitModel() + { + throw new NotImplementedException(); + } + + public override bool Esitmate(Mat bgrMat, List voloResultBox, out List points) + { + throw new NotImplementedException(); + } + + public override void DisposeModel() + { + throw new NotImplementedException(); + } +} \ No newline at end of file diff --git a/Assets/Scripts/PoseCheck/EstimateModel/MediaPipeEstimator.cs.meta b/Assets/Scripts/PoseCheck/EstimateModel/MediaPipeEstimator.cs.meta new file mode 100644 index 0000000..f686e8a --- /dev/null +++ b/Assets/Scripts/PoseCheck/EstimateModel/MediaPipeEstimator.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 5218b8c2234cf5145a26a3eb202cafd2 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Scripts/PoseCheck/EstimateModel/OpenPoseEsimater.cs b/Assets/Scripts/PoseCheck/EstimateModel/OpenPoseEsimater.cs new file mode 100644 index 0000000..cb670a6 --- /dev/null +++ b/Assets/Scripts/PoseCheck/EstimateModel/OpenPoseEsimater.cs @@ -0,0 +1,89 @@ +using OpenCVForUnity.CoreModule; +using OpenCVForUnity.DnnModule; +using OpenCVForUnity.ImgprocModule; +using OpenCVForUnity.UnityUtils; +using System; +using System.Collections.Generic; +using UnityEngine; +using Yoga; + +public class OpenPoseEsimater : Estimator +{ + KeypointsModel _openPoseModel; + private Net _net; + + private double threshold = 0.5; + public override void InitModel() + { + _net = null; + var modelFilePath = Utils.getFilePath(YogaConfig.MODEL_PATHS[ModelType.OpenPose]); + if (string.IsNullOrEmpty(modelFilePath)) + { + Debug.LogError("modelFilePath is empty. Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); + return; + } + + _net = Dnn.readNet(modelFilePath); + + _openPoseModel = new KeypointsModel(_net); + _openPoseModel.setInputScale(new Scalar(YogaConfig.InScale)); + _openPoseModel.setInputSize(new Size(YogaConfig.InWidth, YogaConfig.InHeight)); + _openPoseModel.setInputMean(new Scalar(YogaConfig.InMean)); + _openPoseModel.setInputSwapRB(false); + _openPoseModel.setInputCrop(false); + } + + public override void DisposeModel() + { + if (_net != null) + _net.Dispose(); + } + + public override bool Esitmate(Mat bgrMat, List voloResultBox, out List points) + { + points = null; + + if (_openPoseModel == null) + throw new ArgumentException("CVEstimator.Init: args[1] is not KeypointsModel") +; + OpenCVForUnity.CoreModule.Rect roiRect = new OpenCVForUnity.CoreModule.Rect( + (int)voloResultBox[0][0], + (int)voloResultBox[0][1], + Math.Abs((int)(voloResultBox[0][2] - voloResultBox[0][0])), + Math.Abs((int)(voloResultBox[0][3] - voloResultBox[0][1]))); + + if (roiRect.y < 0 || //0 <= _rowRange.start + (roiRect.y + roiRect.height) < roiRect.y || // _rowRange.start <= _rowRange.end + bgrMat.rows() < (roiRect.y + roiRect.height)) //_rowRange.end <= m.rows + return false; //重新检测 + + Mat personRectImg = new Mat(bgrMat, roiRect);//获取人体区域 + points = _openPoseModel.estimate(personRectImg, (float)threshold).toList(); + //将人体区域的坐标转换为原图坐标 + for (int j = 0; j < points.Count; j++) + { + if (points[j] == null || + (points[j].x == -1 && points[j].y == -1)) //没找到的点，跳过 + continue; + + points[j].x += roiRect.x; + points[j].y += roiRect.y; + } + + if (!YogaManager.Instance.ActionCheckPoints(points)) + { + Debug.Log("ActionCheckPoints failed. Re-Estimation."); + return false; //重新检测 + } + return true; + } + + public override void Check(ref Mat img) + { + if (_net == null) + { + Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false); + Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false); + } + } +} diff --git a/Assets/Scripts/PoseCheck/EstimateModel/OpenPoseEsimater.cs.meta b/Assets/Scripts/PoseCheck/EstimateModel/OpenPoseEsimater.cs.meta new file mode 100644 index 0000000..799b5c4 --- /dev/null +++ b/Assets/Scripts/PoseCheck/EstimateModel/OpenPoseEsimater.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 74f91c9ecee115b44bc9368fe1e41afd +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/Scripts/PoseCheck/MotionCaptureManager.cs b/Assets/Scripts/PoseCheck/MotionCaptureManager.cs index ce3cb28..698bd06 100644 --- a/Assets/Scripts/PoseCheck/MotionCaptureManager.cs +++ b/Assets/Scripts/PoseCheck/MotionCaptureManager.cs @@ -1,11 +1,6 @@ -#undef DEBUG_MODE -#define DEBUG_MODE - - -using OpenCVForUnity.CoreModule; +using OpenCVForUnity.CoreModule; using OpenCVForUnity.DnnModule; using OpenCVForUnity.ImgprocModule; -using OpenCVForUnity.ObjdetectModule; using OpenCVForUnity.UnityUtils; using OpenCVForUnity.UnityUtils.Helper; using OpenCVForUnityExample; @@ -26,27 +21,13 @@ namespace Yoga private bool isInited = false; private Texture2D texture; - private Net _net; - private Dictionary _models = new Dictionary(); - - private KeypointsModel _openPoseModel; - - public int inpWidth = 416; - public int inpHeight = 416; - public float confThreshold = 0.35f; - public float nmsThreshold = 0.6f; - public int topK = 1000; - public float SamplingRate = 0.3f; private List _voloResult = new List(); private List _currPersonPoints = new List(); - private YOLOv7ObjectDetector _objectDetector; private Mat _bgrMat; - private double threshold = 0.5; - private bool _isOnCamCapture = false; public bool IsOnCamCapture { get => _isOnCamCapture; internal set => _isOnCamCapture = value; } public WebCamTextureToMatHelper WebCamTextureToMatHelper => _webCamTextureToMatHelper; @@ -82,45 +63,28 @@ namespace Yoga _webCamTextureToMatHelper = gameObject.GetComponent(); _webCamTextureToMatHelper.Initialize(); - YogaManager.Instance.InitData(); + UIManager.Instance.LoadReset(); + + if (YogaManager.Instance.Action == null) + YogaManager.Instance.InitData(); - - UIManager.Instance.Init(); UIManager.Instance.ShowPanel(false, YogaManager.Instance.Action); - LoadModels();//加载模型 + Utils.setDebugMode(true); //打印日志 + //LoadModels();//加载模型 - CVEstimator.Instance.Init(_objectDetector, _openPoseModel);//初始化姿态检测 - } - - private void LoadModels() - { - //Utils.setDebugMode(true); //打印日志 - _net = null; - - //先用YOLOv7检测人体，再用OpenPose检测人体姿态 - _objectDetector = new YOLOv7ObjectDetector( - Utils.getFilePath("OpenCVForUnity/dnn/yolov7-tiny.weights"), - Utils.getFilePath("OpenCVForUnity/dnn/yolov7-tiny.cfg"), - Utils.getFilePath("OpenCVForUnity/dnn/coco.names"), - new Size(inpWidth, inpHeight), confThreshold, nmsThreshold/*, topK*/); - - - var modelFilePath = Utils.getFilePath(YogaConfig.MODEL_PATHS[ModelType.OpenPose]); - if (string.IsNullOrEmpty(modelFilePath)) + switch (YogaManager.Instance.Action.ModelType) { - Debug.LogError("modelFilePath is empty. Please copy from “OpenCVForUnity/StreamingAssets/” to “Assets/StreamingAssets/” folder. "); - return; + case ModelType.OpenPose: + YogaManager.Instance.CurrentEstimator = new OpenPoseEsimater(); + break; + case ModelType.Mediapipe: + YogaManager.Instance.CurrentEstimator = new MediaPipeEstimator(); + break; } + YogaManager.Instance.CurrentEstimator.Init(); - _net = Dnn.readNet(modelFilePath); - - _openPoseModel = new KeypointsModel(_net); - _openPoseModel.setInputScale(new Scalar(YogaConfig.InScale)); - _openPoseModel.setInputSize(new Size(YogaConfig.InWidth, YogaConfig.InHeight)); - _openPoseModel.setInputMean(new Scalar(YogaConfig.InMean)); - _openPoseModel.setInputSwapRB(false); - _openPoseModel.setInputCrop(false); + CVEstimator.Instance.Init();//初始化姿态检测 } private void Update() @@ -137,14 +101,12 @@ namespace Yoga Mat img = _webCamTextureToMatHelper.GetMat(); Imgproc.cvtColor(img, img, Imgproc.COLOR_BGR2RGB); _rgbaMat = img.clone(); - if (_net == null) - { - Imgproc.putText(img, "model file is not loaded.", new Point(5, img.rows() - 30), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false); - Imgproc.putText(img, "Please read console message.", new Point(5, img.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.7, new Scalar(255, 255, 255), 2, Imgproc.LINE_AA, false); - } + + YogaManager.Instance.CurrentEstimator.Check(ref img);//检测模型，将错误信息打印在图片上 + if (_voloResult.Count >= 2) - DebugPrintObjectLayout(img, _voloResult[0], _voloResult[1], _voloResult[2]); + YogaManager.Instance.CurrentEstimator.DebugPrintObjectLayout(img, _voloResult[0], _voloResult[1], _voloResult[2]); if (_currPersonPoints != null && _currPersonPoints.Count > 0) { @@ -218,7 +180,6 @@ namespace Yoga } AvatarAction actionType = (AvatarAction)args.FirstOrDefault(); - UIManager.Instance.ShowHint(actionType.ToString(), "Follow the robot!"); //检测动作 var startTime = DateTime.Now; @@ -241,37 +202,7 @@ namespace Yoga #endregion - private void DebugPrintObjectLayout(Mat image, float[] box, float[] conf, float[] cls, bool isRGB = false) - { - float left = box[0]; - float top = box[1]; - float right = box[2]; - float bottom = box[3]; - int classId = (int)cls[0]; - - Scalar c = _objectDetector.palette[classId % _objectDetector.palette.Count]; - Scalar color = isRGB ? c : new Scalar(c.val[2], c.val[1], c.val[0], c.val[3]); - - Imgproc.rectangle(image, new Point(left, top), new Point(right, bottom), color, 2); - - string label = String.Format("{0:0.00}", conf[0]); - if (_objectDetector.classNames != null && _objectDetector.classNames.Count != 0) - { - if (classId < (int)_objectDetector.classNames.Count) - { - label = _objectDetector.classNames[classId] + " " + label; - } - } - - int[] baseLine = new int[1]; - Size labelSize = Imgproc.getTextSize(label, Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, 1, baseLine); - - top = Mathf.Max((float)top, (float)labelSize.height); - Imgproc.rectangle(image, new Point(left, top - labelSize.height), - new Point(left + labelSize.width, top + baseLine[0]), color, Core.FILLED); - Imgproc.putText(image, label, new Point(left, top), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, Scalar.all(255), 1, Imgproc.LINE_AA); - } /// /// Raises the destroy event. @@ -280,11 +211,9 @@ namespace Yoga { _webCamTextureToMatHelper.Dispose(); - if (_net != null) - _net.Dispose(); + YogaManager.Instance.CurrentEstimator.Dispose(); _bgrMat.Dispose(); - _models.Clear(); CVEstimator.Instance.Dispose(); } diff --git a/Assets/Scripts/UI/ActionListUIPanel.cs b/Assets/Scripts/UI/ActionListUIPanel.cs index f4c7593..65ce94f 100644 --- a/Assets/Scripts/UI/ActionListUIPanel.cs +++ b/Assets/Scripts/UI/ActionListUIPanel.cs @@ -8,7 +8,14 @@ public class ActionListUIPanel : UIPanelBase public void OnLoadBtnClicked(int index) { YogaManager.Instance.ActionIndex = index; + YogaManager.Instance.InitData(); LoadingManager.Instance.Load("YogaMain"); UIManager.Instance.ClearPanel(); } + + public void OnBackBtnClicked() + { + UIManager.Instance.CloseCurrent(); + UIManager.Instance.ShowPanel(); + } } diff --git a/Assets/Scripts/UI/LoadingManager.cs b/Assets/Scripts/UI/LoadingManager.cs index e06fffa..97ddc61 100644 --- a/Assets/Scripts/UI/LoadingManager.cs +++ b/Assets/Scripts/UI/LoadingManager.cs @@ -17,7 +17,11 @@ public class LoadingManager:MonoSingleton private IEnumerator LoadingScene(string targetScene) { - yield return new WaitForSeconds(3f); + //演示用，正式版需要根据场景大小调整 + if (targetScene.Equals("YogaMain")) + { + yield return new WaitForSeconds(3f); + } var loadNew = SceneManager.LoadSceneAsync(targetScene, LoadSceneMode.Single); loadNew.completed += (AsyncOperation obj2) => { diff --git a/Assets/Scripts/UI/UIManager.cs b/Assets/Scripts/UI/UIManager.cs index 9e0c24a..4202c69 100644 --- a/Assets/Scripts/UI/UIManager.cs +++ b/Assets/Scripts/UI/UIManager.cs @@ -15,6 +15,9 @@ public class UIManager : MonoSingleton public Camera UiCamera { get; set; } + private static bool _isInited = false; + public static bool IsInited { get => _isInited; private set => _isInited = value; } + private RectTransform _uiPanelRoot; private RectTransform _uiPopRoot; @@ -27,6 +30,8 @@ public class UIManager : MonoSingleton public override void Init() { + if (_isInited) + return; //ҵǰscene UiCamera = Camera.main; @@ -99,8 +104,14 @@ public class UIManager : MonoSingleton else _hud = transform.Find("HUDRoot").GetComponent(); _hud.transform.SetSiblingIndex(3); + _isInited = true; } + public void LoadReset() + { + _isInited = false; + Init(); + } private T GetPanel() where T : UIBase { Type type = typeof(T); diff --git a/Assets/Scripts/YogaConfig.cs b/Assets/Scripts/YogaConfig.cs index 8a915fe..c5c9b8d 100644 --- a/Assets/Scripts/YogaConfig.cs +++ b/Assets/Scripts/YogaConfig.cs @@ -139,9 +139,4 @@ public static class YogaConfig public static readonly double[] InMean = new double[] { 128.0, 128.0, 128.0 }; } -public enum ModelType -{ - OpenPose, - YoloV7, - //, -} + diff --git a/Assets/Scripts/YogaDataLoader.cs b/Assets/Scripts/YogaDataLoader.cs index 7637a50..9ee8b61 100644 --- a/Assets/Scripts/YogaDataLoader.cs +++ b/Assets/Scripts/YogaDataLoader.cs @@ -15,6 +15,7 @@ public class YogaDataLoader { VideoPath = "Video/Action2", Action = AvatarAction.Nod, + ModelType = ModelType.OpenPose, MaxActionCount = 1000, TotalSeconds = 20.0f, MustPoints = new List() { "Nose", "REye", "LEye", "Neck" }, @@ -25,6 +26,7 @@ public class YogaDataLoader { VideoPath = "Video/Action1", Action = AvatarAction.HeadShake, + ModelType = ModelType.OpenPose, MaxActionCount = 4, TotalSeconds = 20.0f, MustPoints = new List() { "Nose", "REye", "LEye", "Neck" } @@ -33,6 +35,7 @@ public class YogaDataLoader { VideoPath = "Video/Action2", Action = AvatarAction.Nod, + ModelType = ModelType.OpenPose, MaxActionCount = 4, TotalSeconds = 20.0f, MustPoints = new List() { "Nose", "REye", "LEye", "Neck" } @@ -41,6 +44,7 @@ public class YogaDataLoader { VideoPath = "Video/Action3", Action = AvatarAction.HandsUp, + ModelType = ModelType.OpenPose, MaxActionCount = 4, TotalSeconds = 12.66f, MustPoints = new List() { "Nose", "RShoulder", "LShoulder", "RightWrist" } @@ -61,8 +65,17 @@ public class YogaData { public string VideoPath; public AvatarAction Action; + public ModelType ModelType; public int MaxActionCount; public float TotalSeconds; public List MustPoints = new List(); public List AnyPoints = new List(); +} + +public enum ModelType +{ + OpenPose, + YoloV7, + Mediapipe, + //, } \ No newline at end of file diff --git a/Assets/Scripts/YogaManager.cs b/Assets/Scripts/YogaManager.cs index 2a25eef..e4c8096 100644 --- a/Assets/Scripts/YogaManager.cs +++ b/Assets/Scripts/YogaManager.cs @@ -33,8 +33,15 @@ public class YogaManager : MonoSingleton private Dictionary _actions = new Dictionary(); - public YogaData Action { get => _action; set => _action = value; } + public YogaData Action + { + get => _action; + set => _action = value; + } + public Estimator CurrentEstimator { get => currentEstimator; internal set => currentEstimator = value; } + private YogaData _action = null; + private Estimator currentEstimator; public void InitData() { @@ -126,7 +133,7 @@ public class YogaManager : MonoSingleton { //Debug.LogError($"ActionCheckPoints failed, {p} is not valid"); return false; - } + } } //不设的情况下，不检测 @@ -144,4 +151,4 @@ public class YogaManager : MonoSingleton return true; } -} +} \ No newline at end of file diff --git a/Assets/UI/Img/Arrow Simple Left.png b/Assets/UI/Img/Arrow Simple Left.png new file mode 100644 index 0000000000000000000000000000000000000000..6e40c52f31eb0b528d220912b73789e8fc27cf20 GIT binary patch literal 6163 zcmbVQ2{_bi+kfUimNBv>L~3T7LJVV!ZETYzOWBK3VvMn57%^F6W-8KVIptK8rIJn? zPEI%t4k__;MhugK3u8t`$Jr6aZwLsu=x{qipt&c>Gv0|A744Rm z!1Pb^+Q3MQVOWO~9PDwnDO9K+mdT;xQes(giPRK3!e_ly_$(PVA>clXaANETYa|A7 zn>>ARPV59G&eGVF$RLqOI7@3|W;mT3W)@+DBa_IcCZ_O9A)1n@W+WK-)#J?FR`)TOeAt# zl3*BL0r^|$#0{zOOcOt5B70{7gXxk4-PDyt6Hj$YVA474gbi#q>(8C?`O{?_+057! zrym#{7tT&jH2BK`rZb(xv?IV>BNItvqN&9OGMNg$=43;X8I?r(BI?NwkB&(FTTu#? zO!^m5$eD0DhyGuM!x_{Fc0w#21{@trk7Sy}$3+rw-(aLVv03Z{Xc&5D_SbQDCnui- zc0@D_P9*xdI^x`2oGeVOEi8y+W7E&>dU{gb;}SXaI0n<**^U6gF^-N7r&^m^F~b>5 z2GQKg(u_#4q=XT}A|gmcI@5}7L1soUEtuhdu6JfLc1kE9S^p&+!r2U{nn)qhNnsRg^DuKp#Gl%{6QW^8(pi7iD)A~DYBaT?lb93= zooH=sK_Z5mQy4^Rl64r-nr_Loj0iIikFaJ+X#bf>RJZ6vxW}nq6Um>sbFBZ?ju$%LicgdhpVL{js5qq^(i%X;cG42x-% z`;x6@FD3q#nYmFt>xb}-uhK*rQ=8LEQ;!@ul2a=r#oJ8~`^(q{W0PLvi1VlFWz+WY zD)IPuS<>?$-xhsv_OdZ5bJ4E;fj+j>D!z>`hO-`pi{;2o=ug}-SeU)%u7X;-+nM-$E?vd&y-#eIO7my16adWlfYc&eKv#oa29EuD467 zNA{%eW*5F>&|;A|{|$GhQpgk{?i%z~ zL;PVh(b>bXcdPu1d9pqjfCbS%tmFGqxgDJsRg9!Wg)@(%ZKZh>7p?L7>T?cksq3Y8 zhNRR$yVTXg9|tK&kNnAvlcn97SnxKfJ=h(}K8P%0ux(&e%$hhOkS)i=cQllU^N8x@SfE>Ij|KmY~Hj-qJbfM}>?|fWr zV*ui^mj{QAX75T642^f1tNodVXZ7~PV3&4x7)jJ zm#o&&3!U79&^}#7W}{sFm8Ld4mzrXhlUnBM{D50kKxoyc({HBsWO$9P+HK(gm04C$ z78HbsUFZhj>=wfsrl80H)Jx~bogK`(`TQewO5yfIs}dHx>cK+CAhP7N!@G-Cz)dv|2TTz%zlMDI6R<36EM0)#;{8dk|v;cbD$FyA&YaDs3Cpwa&hYvH1BtUhgU{0M^kLA7W>YU+uiIaj(?PV@fi zKa^%pf?G1;XE*ZoYmuDtqk{sJIk%z~i4trSUcAM8v~%ES(SqE#>q zi}N#AeL-!f$LN%(*~$LZ#?m5(#nLpJ%Z*oRb8SV%*}I;Jml}XBy}5U#uY=vzE*9!! zZl8as2F4)P+?l7+CEMq<^g);IsgZ-?mQskq#al)2oxX&*>{aifB0&I_H@M|+{ZO7r zOl^NnL$$GDYUBi;eaW2Z5GzvTgj-)%=OsDYKjvc6A1|yMcl4-OLNhCD6Tc7^Y%>?= zO4BAU-M=MV{L73|m_RIeXMC1}=_ulDH(9rLIclUpG-;A}MYr;W(ADOdn0R^#Efc0vAip$S zgV6f~i>p$#5VuP>kyVQ{>-7%60H2x(j3=WEE`xV&i!%%Er{|S>bLvV0K*tu(r5~sI za*6~W)If(vs{khYq1@1rQQt|~GFRTnrF-^WJ8~0staa`cg=lBO0_}6yTxh}_81&Xi zxiEwF1@GC!sVBM3E%Q4@kA*nd-_{48t}5EJYeP^^4=SxgwcYq2ED#M3&9K+Xrr+@c z?*|hkpnK)>wrPf1%_d%$UP8gr69PCWZJmI?^ zRjoK1<~KR1O|2U6+&b z?U@Bg|6Ry6f?y4dq|{O@ArV`k5^9h4pmNH=7Vty^uT~>YyL2v#B-$m zy1@egR+D3Y8BA7%&?=0>7+m7yjsoh4ZsT(GpuJhV0EArGj)2h}{Xt=Q3MNv^W$a%C z=TNrnM}cdm(adp^QbHa72ZAVE9eLqrCOyz-r}~oU=%CS>wHIhc{6D9;AP)To3vF&3iM=-h2hHzY0uTb({x2`^#jY0A>`v}W&I)~G~%%;H*CUI4RL`) zzq0r6Z8fB8v3@=h8ppkJWm%s1z=UnTy&BTw3pj&INB{A54pwc>@-9syulF{ni%HT` zbGIk!h|m;f!73`z2Knu|PX~Gfa8xPo{cS%Kmk5&ITux&t;uP?y^mL>vg72 zChGX1q-h_LLi5lCAu7fV0SK1=gFhIk(V0eS_GPJu8lvnJYmXzi(pX`ux^ zJmT+&F@h7T;Y8ttRkd=u%Xzdju?(E=voMk(2nfp>?q&b18f7KT%ZnBI{?Uvlw>05R)sb`~K|K`AUSH)ds<4;F-U^F14vYi~n(4m*|=sDb%*k5B=us>CKoS{_>A z42a6EHJxU|qzl6K>~SpfRRgUxOK!wkrD1vwI+mHj*LSv4y?kye%wyH}NoA%SgDA7h zO}A+8w@ET}Zw7z(m?-NgEMO5HhWzHmdtSZJO|$8DL$z&v%!rgFjLQw5_Kk$6o8j}( zhA@^l?brb;A3*6#VzE*_J91B?IGlUC2p-6ES(4-ERkVSW{dlmOJ3NnTj!ZM|qV=V= zSAeKvr-aF}gnbPJr~>bUmZr^)-nm6no7oYHpGpedlZ<}?HAGupQ_We1$opWzD?wCM zudt*cvyc1}Rr|)a(e3b(;0J2Ey+-wS3|RALHesSx=JS!Vxunn#+yPtD6!oO;YR9rL zEKlH0AXG3Vr91~%+3SVN?6`?Ka6vx%kk_iHN?&jG+tY5<8r<4m)w);7Kc~rg5a6VFIWK(0! z=cd4&z)e||DG&3IGFuzoYo0(-WN}kV$yp7{)Bv-KpOK79z!mv{dgHT3dk{6rj99wp7=Ff?Fw z@>aQ}J^O~Hf-eL4abtIkm+W7LL;@whhQ>04mR3_djh`^aVxai@?Uqbi#UAf|v@^Wl z-(Lq z8e2+h_LzHamOC3VA13K~{)kzT#A}Ata|P_>s;Hg5HqyM5?tQ1Oe#G*hiS|FTH4XSeyn0(ow73+xa*YmeC;I{<}F4|0z{iPc%Ojj}mw`NcfT)t`p4;2y8) zKB$8mv6O2633e}8P~=KqA+Ka6vK-`Fq@ZS}-RjbjE!BY;P2*FKo#Foy3I?U9I_h1! z6=u}tO1Q?*-6I*o#(oy#?vYKCeO~1+>U&`y+0zcAb Si|GgO