using OpenCVForUnity.CoreModule; using OpenCVForUnity.ImgcodecsModule; using OpenCVForUnity.ImgprocModule; using OpenCVForUnity.UnityUtils; using OpenCVForUnity.VideoioModule; using System; using System.Collections; using UnityEngine; using UnityEngine.SceneManagement; using UnityEngine.UI; namespace OpenCVForUnityExample { /// /// VideoCapture Camera Input Example /// An example of input a camera stream using the VideoCapture class. /// Works well on Windows and MacOS platforms. /// public class VideoCaptureCameraInputExample : MonoBehaviour { /// /// The open botton. /// public Button openButton; /// /// The release botton. /// public Button releaseButton; /// /// The requested device id dropdown. /// public Dropdown requestedDeviceIdDropdown; /// /// The requested device id. /// public DeviceIdPreset requestedDeviceId = DeviceIdPreset._0; /// /// The requested resolution dropdown. /// public Dropdown requestedResolutionDropdown; /// /// The requested resolution. /// public ResolutionPreset requestedResolution = ResolutionPreset._640x480; /// /// The requested FPS dropdown. /// public Dropdown requestedFPSDropdown; /// /// The requested FPS. /// public FPSPreset requestedFPS = FPSPreset._30; /// /// The requested FOURCC dropdown. /// public Dropdown requestedFOURCCDropdown; /// /// The requested FOURCC. /// public FOURCCPreset requestedFOURCC = FOURCCPreset.None; /// /// The requested CONVERT_RGB toggle. /// public Toggle requestedCONVERT_RGBToggle; /// /// The videocapture. /// VideoCapture capture; /// /// The input mat. /// Mat inputMat; /// /// The rgb mat. /// Mat rgbMat; /// /// The texture. /// Texture2D texture; /// /// Indicates whether the video is playing. /// bool isPlaying = false; /// /// The requested width. /// int requestedWidth = 640; /// /// The requested height. /// int requestedHeight = 480; /// /// The prop FOURCC. /// FOURCCPreset propFOURCC = FOURCCPreset.None; /// /// The prop ConvertRGB. /// bool propConvertRGB = true; /// /// The FPS monitor. /// FpsMonitor fpsMonitor; // Use this for initialization IEnumerator Start() { fpsMonitor = GetComponent(); // Tricks for Adding camera permission settings to AndroidManifest.xml. Debug.Log(WebCamTexture.devices.Length); // Checks user camera permission state for Android and iOS. yield return RequestUserCameraPermission(); capture = new VideoCapture(); // Update GUI state requestedDeviceIdDropdown.value = Array.IndexOf(System.Enum.GetNames(typeof(DeviceIdPreset)), requestedDeviceId.ToString()); requestedResolutionDropdown.value = (int)requestedResolution; requestedFPSDropdown.value = Array.IndexOf(System.Enum.GetNames(typeof(FPSPreset)), requestedFPS.ToString()); requestedFOURCCDropdown.value = Array.IndexOf(System.Enum.GetNames(typeof(FOURCCPreset)), requestedFOURCC.ToString()); requestedCONVERT_RGBToggle.isOn = true; releaseButton.interactable = false; Dimensions(requestedResolution, out requestedWidth, out requestedHeight); } protected virtual IEnumerator RequestUserCameraPermission() { #if UNITY_IOS && UNITY_2018_1_OR_NEWER UserAuthorization mode = UserAuthorization.WebCam; if (!Application.HasUserAuthorization(mode)) { isUserRequestingPermission = true; yield return Application.RequestUserAuthorization(mode); float timeElapsed = 0; while (isUserRequestingPermission) { if (timeElapsed > 0.25f) { isUserRequestingPermission = false; break; } timeElapsed += Time.deltaTime; yield return null; } } if (!Application.HasUserAuthorization(mode)) { if (fpsMonitor != null) { fpsMonitor.consoleText = "Camera permission is denied."; } yield break; } #elif UNITY_ANDROID && UNITY_2018_3_OR_NEWER string permission = UnityEngine.Android.Permission.Camera; if (!UnityEngine.Android.Permission.HasUserAuthorizedPermission(permission)) { isUserRequestingPermission = true; UnityEngine.Android.Permission.RequestUserPermission(permission); float timeElapsed = 0; while (isUserRequestingPermission) { if (timeElapsed > 0.25f) { isUserRequestingPermission = false; break; } timeElapsed += Time.deltaTime; yield return null; } } if (!UnityEngine.Android.Permission.HasUserAuthorizedPermission(permission)) { if (fpsMonitor != null) { fpsMonitor.consoleText = "Camera permission is denied."; } yield break; } #else yield return true; #endif } #if (UNITY_IOS && UNITY_2018_1_OR_NEWER) || (UNITY_ANDROID && UNITY_2018_3_OR_NEWER) bool isUserRequestingPermission; IEnumerator OnApplicationFocus(bool hasFocus) { yield return null; if (isUserRequestingPermission && hasFocus) isUserRequestingPermission = false; } #endif // Update is called once per frame void Update() { if (isPlaying) { if (capture.grab()) { capture.retrieve(inputMat); cvtColor(inputMat, rgbMat, propFOURCC, propConvertRGB); Utils.matToTexture2D(rgbMat, texture); } } } protected void cvtColor(Mat src, Mat dst, FOURCCPreset FOURCC, bool convertRGB) { if (convertRGB) { src.copyTo(dst); return; } switch (FOURCC) { case FOURCCPreset.MJPG: dst = Imgcodecs.imdecode(src, Imgcodecs.IMREAD_COLOR); break; case FOURCCPreset.RGB3: src.copyTo(dst); break; case FOURCCPreset.BGR3: Imgproc.cvtColor(src, dst, Imgproc.COLOR_BGR2RGB); break; case FOURCCPreset.GRAY: Imgproc.cvtColor(src, dst, Imgproc.COLOR_GRAY2RGB); break; case FOURCCPreset.NV21: if (src.channels() != 1) src = src.reshape(1, dst.rows() + (dst.rows() / 2)); Imgproc.cvtColor(src, dst, Imgproc.COLOR_YUV2RGB_NV21); break; case FOURCCPreset.YV12: if (src.channels() != 1) src = src.reshape(1, dst.rows() + (dst.rows() / 2)); Imgproc.cvtColor(src, dst, Imgproc.COLOR_YUV2RGB_YV12); break; case FOURCCPreset.YUYV: if (src.channels() != 2) src = src.reshape(2, dst.rows()); Imgproc.cvtColor(src, dst, Imgproc.COLOR_YUV2RGB_YUYV); break; case FOURCCPreset.YUY2: if (src.channels() != 2) src = src.reshape(2, dst.rows()); Imgproc.cvtColor(src, dst, Imgproc.COLOR_YUV2RGB_YUY2); break; case FOURCCPreset.NV12: if (src.channels() != 1) src = src.reshape(1, dst.rows() + (dst.rows() / 2)); Imgproc.cvtColor(src, dst, Imgproc.COLOR_YUV2RGB_NV12); break; case FOURCCPreset.H264: src.copyTo(dst); break; default: src.copyTo(dst); break; } } /// /// Raises the destroy event. /// void OnDestroy() { capture.release(); if (inputMat != null) inputMat.Dispose(); if (rgbMat != null) rgbMat.Dispose(); Texture2D.Destroy(texture); } /// /// Raises the back button click event. /// public void OnBackButtonClick() { SceneManager.LoadScene("OpenCVForUnityExample"); } /// /// Raises the open button click event. /// public void OnOpenButtonClick() { if (capture.isOpened()) return; capture.open((int)requestedDeviceId); if (!capture.isOpened()) { Debug.LogError("capture.isOpened() is false. " + "DeviceId:" + (int)requestedDeviceId); if (fpsMonitor != null) fpsMonitor.consoleText = "capture.isOpened() is false. " + "DeviceId:" + (int)requestedDeviceId; capture.release(); return; } capture.set(Videoio.CAP_PROP_FRAME_WIDTH, requestedWidth); capture.set(Videoio.CAP_PROP_FRAME_HEIGHT, requestedHeight); capture.set(Videoio.CAP_PROP_FPS, (int)requestedFPS); switch (requestedFOURCC) { case FOURCCPreset.None: // break; case FOURCCPreset.MJPG: capture.set(Videoio.CAP_PROP_FOURCC, VideoWriter.fourcc('M', 'J', 'P', 'G')); break; case FOURCCPreset.RGB3: capture.set(Videoio.CAP_PROP_FOURCC, VideoWriter.fourcc('R', 'G', 'B', '3')); break; case FOURCCPreset.BGR3: capture.set(Videoio.CAP_PROP_FOURCC, VideoWriter.fourcc('B', 'G', 'R', '3')); break; case FOURCCPreset.GRAY: capture.set(Videoio.CAP_PROP_FOURCC, VideoWriter.fourcc('G', 'R', 'A', 'Y')); break; case FOURCCPreset.NV21: capture.set(Videoio.CAP_PROP_FOURCC, VideoWriter.fourcc('N', 'V', '2', '1')); break; case FOURCCPreset.YV12: capture.set(Videoio.CAP_PROP_FOURCC, VideoWriter.fourcc('Y', 'V', '1', '2')); break; case FOURCCPreset.YUYV: capture.set(Videoio.CAP_PROP_FOURCC, VideoWriter.fourcc('Y', 'U', 'Y', 'V')); break; case FOURCCPreset.YUY2: capture.set(Videoio.CAP_PROP_FOURCC, VideoWriter.fourcc('Y', 'U', 'Y', '2')); break; case FOURCCPreset.NV12: capture.set(Videoio.CAP_PROP_FOURCC, VideoWriter.fourcc('N', 'V', '1', '2')); break; case FOURCCPreset.H264: capture.set(Videoio.CAP_PROP_FOURCC, VideoWriter.fourcc('H', '2', '6', '4')); break; default: // break; } if (!requestedCONVERT_RGBToggle.isOn) { capture.set(Videoio.CAP_PROP_CONVERT_RGB, 0); } else { capture.set(Videoio.CAP_PROP_CONVERT_RGB, 1); } Debug.Log("CAP_PROP_FORMAT: " + capture.get(Videoio.CAP_PROP_FORMAT)); Debug.Log("CAP_PROP_FPS: " + capture.get(Videoio.CAP_PROP_FPS)); Debug.Log("CAP_PROP_FRAME_WIDTH: " + capture.get(Videoio.CAP_PROP_FRAME_WIDTH)); Debug.Log("CAP_PROP_FRAME_HEIGHT: " + capture.get(Videoio.CAP_PROP_FRAME_HEIGHT)); double ext = capture.get(Videoio.CAP_PROP_FOURCC); Debug.Log("CAP_PROP_FOURCC: " + (char)((int)ext & 0XFF) + (char)(((int)ext & 0XFF00) >> 8) + (char)(((int)ext & 0XFF0000) >> 16) + (char)(((int)ext & 0XFF000000) >> 24)); Debug.Log("CAP_PROP_CONVERT_RGB: " + capture.get(Videoio.CAP_PROP_CONVERT_RGB)); Debug.Log("CAP_PROP_MODE: " + capture.get(Videoio.CAP_PROP_MODE)); Debug.Log("CAP_PROP_BRIGHTNESS: " + capture.get(Videoio.CAP_PROP_BRIGHTNESS)); Debug.Log("CAP_PROP_CONTRAST: " + capture.get(Videoio.CAP_PROP_CONTRAST)); Debug.Log("CAP_PROP_SATURATION: " + capture.get(Videoio.CAP_PROP_SATURATION)); Debug.Log("CAP_PROP_HUE: " + capture.get(Videoio.CAP_PROP_HUE)); Debug.Log("CAP_PROP_GAIN: " + capture.get(Videoio.CAP_PROP_GAIN)); Debug.Log("CAP_PROP_EXPOSURE: " + capture.get(Videoio.CAP_PROP_EXPOSURE)); Debug.Log("CAP_PROP_AUTO_EXPOSURE: " + capture.get(Videoio.CAP_PROP_AUTO_EXPOSURE)); Debug.Log("CAP_PROP_GAMMA: " + capture.get(Videoio.CAP_PROP_GAMMA)); Debug.Log("CAP_PROP_ZOOM: " + capture.get(Videoio.CAP_PROP_ZOOM)); Debug.Log("CAP_PROP_FOCUS: " + capture.get(Videoio.CAP_PROP_FOCUS)); Debug.Log("CAP_PROP_AUTOFOCUS: " + capture.get(Videoio.CAP_PROP_AUTOFOCUS)); Debug.Log("CAP_PROP_BUFFERSIZE: " + capture.get(Videoio.CAP_PROP_BUFFERSIZE)); Debug.Log("CAP_PROP_SAR_NUM: " + capture.get(Videoio.CAP_PROP_SAR_NUM)); Debug.Log("CAP_PROP_SAR_DEN: " + capture.get(Videoio.CAP_PROP_SAR_DEN)); Debug.Log("CAP_PROP_BACKEND: " + capture.get(Videoio.CAP_PROP_BACKEND)); Debug.Log("CAP_PROP_AUTO_WB: " + capture.get(Videoio.CAP_PROP_AUTO_WB)); Debug.Log("CAP_PROP_WB_TEMPERATURE: " + capture.get(Videoio.CAP_PROP_WB_TEMPERATURE)); Debug.Log("CAP_PROP_CODEC_PIXEL_FORMAT: " + capture.get(Videoio.CAP_PROP_CODEC_PIXEL_FORMAT)); inputMat = new Mat(); int grabFrameCount = 0; while (!(capture.grab() && capture.retrieve(inputMat))) { if (grabFrameCount > 30) { Debug.LogError("The grab() and retrieve(() method is not work."); if (fpsMonitor != null) fpsMonitor.consoleText = "The grab() and retrieve(() method is not work."; capture.release(); break; } grabFrameCount++; } if (inputMat.width() == 0 && inputMat.height() == 0) { Debug.LogError("The inputMat size returned was 0x0."); if (fpsMonitor != null) fpsMonitor.consoleText = "The inputMat size returned was 0x0."; capture.release(); return; } #if UNITY_IOS int frameWidth = (int)capture.get(Videoio.CAP_PROP_FRAME_HEIGHT); int frameHeight = (int)capture.get(Videoio.CAP_PROP_FRAME_WIDTH); #else int frameWidth = (int)capture.get(Videoio.CAP_PROP_FRAME_WIDTH); int frameHeight = (int)capture.get(Videoio.CAP_PROP_FRAME_HEIGHT); #endif rgbMat = new Mat(frameHeight, frameWidth, CvType.CV_8UC3); texture = new Texture2D(frameWidth, frameHeight, TextureFormat.RGB24, false); gameObject.transform.localScale = new Vector3((float)frameWidth, (float)frameHeight, 1); float widthScale = (float)Screen.width / (float)frameWidth; float heightScale = (float)Screen.height / (float)frameHeight; if (widthScale < heightScale) { Camera.main.orthographicSize = ((float)frameWidth * (float)Screen.height / (float)Screen.width) / 2; } else { Camera.main.orthographicSize = (float)frameHeight / 2; } gameObject.GetComponent().material.mainTexture = texture; string strFOURCC = "" + (char)((int)ext & 0XFF) + (char)(((int)ext & 0XFF00) >> 8) + (char)(((int)ext & 0XFF0000) >> 16) + (char)(((int)ext & 0XFF000000) >> 24); switch (strFOURCC) { case "MJPG": propFOURCC = FOURCCPreset.MJPG; break; case "RGB3": propFOURCC = FOURCCPreset.RGB3; break; case "BGR3": propFOURCC = FOURCCPreset.BGR3; break; case "GRAY": propFOURCC = FOURCCPreset.GRAY; break; case "NV21": propFOURCC = FOURCCPreset.NV21; break; case "YV12": propFOURCC = FOURCCPreset.YV12; break; case "YUYV": propFOURCC = FOURCCPreset.YUYV; break; case "YUY2": propFOURCC = FOURCCPreset.YUY2; break; case "NV12": propFOURCC = FOURCCPreset.NV12; break; case "H264": propFOURCC = FOURCCPreset.H264; break; default: //propFOURCC = FOURCCPreset.None; propFOURCC = requestedFOURCC; break; } if (capture.get(Videoio.CAP_PROP_CONVERT_RGB) == 1) { propConvertRGB = true; } else { propConvertRGB = false; } // The frame size returned by VideoCapture property may differ from the returned Mat. // On iOS platform, the height and width of the frame size are swapped. if (fpsMonitor != null) { fpsMonitor.consoleText = ""; fpsMonitor.Add("getBackendName", capture.getBackendName()); fpsMonitor.Add("CAP_PROP_FORMAT", capture.get(Videoio.CAP_PROP_FORMAT).ToString()); fpsMonitor.Add("CAP_PROP_FPS", capture.get(Videoio.CAP_PROP_FPS).ToString()); fpsMonitor.Add("CAP_PROP_FRAME_WIDTH", capture.get(Videoio.CAP_PROP_FRAME_WIDTH).ToString()); fpsMonitor.Add("CAP_PROP_FRAME_HEIGHT", capture.get(Videoio.CAP_PROP_FRAME_HEIGHT).ToString()); fpsMonitor.Add("CAP_PROP_FOURCC", "" + (char)((int)ext & 0XFF) + (char)(((int)ext & 0XFF00) >> 8) + (char)(((int)ext & 0XFF0000) >> 16) + (char)(((int)ext & 0XFF000000) >> 24)); fpsMonitor.Add("CAP_PROP_CONVERT_RGB", capture.get(Videoio.CAP_PROP_CONVERT_RGB).ToString()); fpsMonitor.Add("inputMat", "size:" + inputMat.width() + "x" + inputMat.height() + " type:" + CvType.typeToString(inputMat.type())); } Utils.setDebugMode(true, true); try { cvtColor(inputMat, rgbMat, propFOURCC, propConvertRGB); Utils.matToTexture2D(rgbMat, texture); openButton.interactable = requestedDeviceIdDropdown.interactable = requestedResolutionDropdown.interactable = requestedFPSDropdown.interactable = requestedFOURCCDropdown.interactable = requestedCONVERT_RGBToggle.interactable = false; releaseButton.interactable = true; isPlaying = true; } catch (Exception e) { Debug.LogError("CVException: " + e); if (fpsMonitor != null) { fpsMonitor.consoleText = "CVException: " + e; } capture.release(); } Utils.setDebugMode(false); } /// /// Raises the release button click event. /// public void OnReleaseButtonClick() { if (!capture.isOpened()) return; capture.release(); if (inputMat != null) inputMat.Dispose(); if (rgbMat != null) rgbMat.Dispose(); Texture2D.Destroy(texture); openButton.interactable = requestedDeviceIdDropdown.interactable = requestedResolutionDropdown.interactable = requestedFPSDropdown.interactable = requestedFOURCCDropdown.interactable = requestedCONVERT_RGBToggle.interactable = true; releaseButton.interactable = false; isPlaying = false; if (fpsMonitor != null) { fpsMonitor.consoleText = ""; } } /// /// Raises the requested device id dropdown value changed event. /// public void OnRequestedDeviceIdDropdownValueChanged(int result) { string[] enumNames = Enum.GetNames(typeof(DeviceIdPreset)); byte value = (byte)System.Enum.Parse(typeof(DeviceIdPreset), enumNames[result], true); if ((byte)requestedDeviceId != value) { requestedDeviceId = (DeviceIdPreset)value; } //Debug.Log((int)requestedDeviceId); } /// /// Raises the requested resolution dropdown value changed event. /// public void OnRequestedResolutionDropdownValueChanged(int result) { if ((int)requestedResolution != result) { requestedResolution = (ResolutionPreset)result; Dimensions(requestedResolution, out requestedWidth, out requestedHeight); } //Debug.Log((int)requestedWidth + " " + (int)requestedHeight); } /// /// Raises the requested FOURCC dropdown value changed event. /// public void OnRequestedFOURCCDropdownValueChanged(int result) { string[] enumNames = Enum.GetNames(typeof(FOURCCPreset)); byte value = (byte)System.Enum.Parse(typeof(FOURCCPreset), enumNames[result], true); if ((byte)requestedFOURCC != value) { requestedFOURCC = (FOURCCPreset)value; } //Debug.Log((int)requestedFOURCC); } /// /// Raises the requestedFPS dropdown value changed event. /// public void OnRequestedFPSDropdownValueChanged(int result) { string[] enumNames = Enum.GetNames(typeof(FPSPreset)); int value = (int)System.Enum.Parse(typeof(FPSPreset), enumNames[result], true); if ((int)requestedFPS != value) { requestedFPS = (FPSPreset)value; } //Debug.Log((int)requestedFPS); } /// /// Raises the Requested CONVERT_RGB toggle value changed event. /// public void OnRequestedCONVERT_RGBToggleValueChanged() { //Debug.Log(requestedCONVERT_RGBToggle.isOn); } public enum DeviceIdPreset : byte { _0 = 0, _1, _2, _3, _4, } public enum FPSPreset : int { _0 = 0, _1 = 1, _5 = 5, _10 = 10, _15 = 15, _30 = 30, _60 = 60, } public enum ResolutionPreset : byte { _50x50 = 0, _640x480, _1280x720, _1920x1080, _9999x9999, } public enum FOURCCPreset : byte { None = 0, MJPG, RGB3, BGR3, GRAY, NV21, YV12, YUYV, YUY2, NV12, H264, } private void Dimensions(ResolutionPreset preset, out int width, out int height) { switch (preset) { case ResolutionPreset._50x50: width = 50; height = 50; break; case ResolutionPreset._640x480: width = 640; height = 480; break; case ResolutionPreset._1280x720: width = 1280; height = 720; break; case ResolutionPreset._1920x1080: width = 1920; height = 1080; break; case ResolutionPreset._9999x9999: width = 9999; height = 9999; break; default: width = height = 0; break; } } } }