using OpenCVForUnity.CoreModule; using OpenCVForUnity.ImgprocModule; using OpenCVForUnity.VideoioModule; using System; using System.Collections; using UnityEngine; using UnityEngine.Events; using UnityEngine.Serialization; namespace OpenCVForUnity.UnityUtils.Helper { /// /// VideoCapture to mat helper. /// v 1.0.4 /// public class VideoCaptureToMatHelper : MonoBehaviour { /// /// Set the video file path. /// [SerializeField, FormerlySerializedAs("requestedVideoFilePath"), TooltipAttribute("Set the video file path.")] protected string _requestedVideoFilePath = string.Empty; public virtual string requestedVideoFilePath { get { return _requestedVideoFilePath; } set { if (_requestedVideoFilePath != value) { _requestedVideoFilePath = value; if (hasInitDone) Initialize(); } } } /// /// Set the apiPreference. VideoCapture API backends identifier. (Advanced Option) /// See ReadMe.pdf for setup instructions for using CAP_FFMPEG on Windows platforms. /// [SerializeField, FormerlySerializedAs("apiPreference"), TooltipAttribute("Set the apiPreference. VideoCapture API backends identifier. (Advanced Option)\n" + "0 = CAP_ANY (Auto detect)[default]\n" + "1400 = CAP_MSMF (Microsoft Media Foundation (via videoInput))\n" + "1900 = CAP_FFMPEG (Open and record video file or stream using the FFMPEG library)\n" + "2200 = CAP_OPENCV_MJPEG (Built-in OpenCV MotionJPEG codec)")] protected int _apiPreference = Videoio.CAP_ANY; public virtual int apiPreference { get { return _apiPreference; } set { if (_apiPreference != value) { _apiPreference = value; if (hasInitDone) Initialize(); } } } /// /// Select the output color format. /// [SerializeField, FormerlySerializedAs("outputColorFormat"), TooltipAttribute("Select the output color format.")] protected ColorFormat _outputColorFormat = ColorFormat.BGR; public virtual ColorFormat outputColorFormat { get { return _outputColorFormat; } set { if (_outputColorFormat != value) { _outputColorFormat = value; if (hasInitDone) Initialize(); } } } /// /// The number of frames before the initialization process times out. /// [SerializeField, FormerlySerializedAs("timeoutFrameCount"), TooltipAttribute("The number of frames before the initialization process times out.")] protected int _timeoutFrameCount = 1500; public virtual int timeoutFrameCount { get { return _timeoutFrameCount; } set { _timeoutFrameCount = (int)Mathf.Clamp(value, 0f, float.MaxValue); } } /// /// Indicates whether to play this video in a loop. /// [SerializeField, FormerlySerializedAs("loop"), TooltipAttribute("Indicates whether to play this video in a loop.")] protected bool _loop = true; public virtual bool loop { get { lock (sync) return _loop; } set { lock (sync) _loop = value; } } /// /// UnityEvent that is triggered when this instance is initialized. /// public UnityEvent onInitialized; /// /// UnityEvent that is triggered when this instance is disposed. /// public UnityEvent onDisposed; /// /// UnityEvent that is triggered when this instance is error Occurred. /// public ErrorUnityEvent onErrorOccurred; /// /// The VideoCapture. /// protected System.Object videoCaptureLockObject = new System.Object(); protected VideoCapture _videoCapture; protected VideoCapture videoCapture { get { lock (videoCaptureLockObject) return _videoCapture; } set { lock (videoCaptureLockObject) _videoCapture = value; } } protected System.Object sync = new System.Object(); protected bool _isReading = false; protected virtual bool isReading { get { lock (sync) return _isReading; } set { lock (sync) _isReading = value; } } protected bool _isPlaying = false; protected virtual bool isPlaying { get { lock (sync) return _isPlaying; } set { lock (sync) _isPlaying = value; } } protected bool _didUpdateThisFrame = false; protected virtual bool didUpdateThisFrame { get { lock (sync) return _didUpdateThisFrame; } set { lock (sync) _didUpdateThisFrame = value; } } protected bool _didUpdateImageBufferInCurrentFrame = false; protected virtual bool didUpdateImageBufferInCurrentFrame { get { lock (sync) return _didUpdateImageBufferInCurrentFrame; } set { lock (sync) _didUpdateImageBufferInCurrentFrame = value; } } /// /// The frame mat. /// protected Mat frameMat; /// /// The base mat. /// protected Mat baseMat; /// /// The image buffer mat. /// protected System.Object imageBufferMatLockObject = new System.Object(); protected Mat _imageBufferMat; protected Mat imageBufferMat { get { lock (imageBufferMatLockObject) return _imageBufferMat; } set { lock (imageBufferMatLockObject) _imageBufferMat = value; } } /// /// The base color format. /// protected ColorFormat baseColorFormat = ColorFormat.BGR; /// /// Indicates whether this instance is waiting for initialization to complete. /// protected bool isInitWaiting = false; /// /// Indicates whether this instance has been initialized. /// protected bool hasInitDone = false; /// /// The initialization coroutine. /// protected IEnumerator initCoroutine; /// /// The get file path coroutine. /// protected IEnumerator getFilePathCoroutine; #if UNITY_WEBGL /// /// The wait frame time coroutine. /// protected IEnumerator waitFrameTimeCoroutine; #endif protected bool _isThreadRunning = false; protected virtual bool isThreadRunning { get { lock (sync) return _isThreadRunning; } set { lock (sync) _isThreadRunning = value; } } protected bool _shouldStopThread = false; protected virtual bool shouldStopThread { get { lock (sync) return _shouldStopThread; } set { lock (sync) _shouldStopThread = value; } } public enum ColorFormat : int { GRAY = 0, RGB, BGR, RGBA, BGRA, } public enum ErrorCode : int { UNKNOWN = 0, VIDEO_FILE_NOT_EXIST, VIDEO_FILE_CANT_OPEN, TIMEOUT, } [Serializable] public class ErrorUnityEvent : UnityEvent { } protected virtual void OnValidate() { _timeoutFrameCount = (int)Mathf.Clamp(_timeoutFrameCount, 0f, float.MaxValue); } protected virtual void LateUpdate() { if (!hasInitDone) return; if (didUpdateThisFrame && !didUpdateImageBufferInCurrentFrame) { #if UNITY_WEBGL ReadFrame(); #elif WINDOWS_UWP || (!UNITY_WSA_10_0 && (NET_4_6 || NET_STANDARD_2_0)) System.Threading.Tasks.Task.Run(() => ReadFrame()); #else System.Threading.ThreadPool.QueueUserWorkItem(_ => ReadFrame()); #endif didUpdateThisFrame = false; } didUpdateImageBufferInCurrentFrame = false; } protected virtual void ReadFrame() { isReading = true; if (isPlaying) { lock (videoCaptureLockObject) { if (_videoCapture != null && !_videoCapture.IsDisposed) { if (_videoCapture.get(Videoio.CAP_PROP_POS_FRAMES) >= _videoCapture.get(Videoio.CAP_PROP_FRAME_COUNT)) { if (loop) { _videoCapture.set(Videoio.CAP_PROP_POS_FRAMES, 0); } else { isPlaying = false; } } if (isPlaying) { lock (imageBufferMatLockObject) { if (_imageBufferMat != null && !_imageBufferMat.IsDisposed) { _videoCapture.read(_imageBufferMat); } } } } } } isReading = false; } /// /// Raises the destroy event. /// protected virtual void OnDestroy() { Dispose(); } /// /// Initializes this instance. /// public virtual void Initialize() { if (isInitWaiting) { CancelInitCoroutine(); ReleaseResources(); } if (onInitialized == null) onInitialized = new UnityEvent(); if (onDisposed == null) onDisposed = new UnityEvent(); if (onErrorOccurred == null) onErrorOccurred = new ErrorUnityEvent(); initCoroutine = _Initialize(); StartCoroutine(initCoroutine); } /// /// Initializes this instance. /// /// Requested video file path. public virtual void Initialize(string requestedVideoFilePath) { if (isInitWaiting) { CancelInitCoroutine(); ReleaseResources(); } _requestedVideoFilePath = requestedVideoFilePath; if (onInitialized == null) onInitialized = new UnityEvent(); if (onDisposed == null) onDisposed = new UnityEvent(); if (onErrorOccurred == null) onErrorOccurred = new ErrorUnityEvent(); initCoroutine = _Initialize(); StartCoroutine(initCoroutine); } /// /// Initializes this instance by coroutine. /// protected virtual IEnumerator _Initialize() { if (hasInitDone) { ReleaseResources(); if (onDisposed != null) onDisposed.Invoke(); } isInitWaiting = true; bool hasFilePathCoroutineCompleted = false; string fullPath = string.Empty; getFilePathCoroutine = Utils.getFilePathAsync(requestedVideoFilePath, (result) => { hasFilePathCoroutineCompleted = true; fullPath = result; }); StartCoroutine(getFilePathCoroutine); int initFrameCount = 0; bool isTimeout = false; while (true) { if (initFrameCount > timeoutFrameCount) { isTimeout = true; break; } else if (hasFilePathCoroutineCompleted) { if (string.IsNullOrEmpty(fullPath)) { videoCapture = null; isInitWaiting = false; initCoroutine = null; getFilePathCoroutine = null; if (onErrorOccurred != null) onErrorOccurred.Invoke(ErrorCode.VIDEO_FILE_NOT_EXIST); yield break; } videoCapture = new VideoCapture(); videoCapture.open(fullPath, apiPreference); if (!videoCapture.isOpened()) { videoCapture.Dispose(); videoCapture = null; isInitWaiting = false; initCoroutine = null; getFilePathCoroutine = null; if (onErrorOccurred != null) onErrorOccurred.Invoke(ErrorCode.VIDEO_FILE_CANT_OPEN); yield break; } imageBufferMat = new Mat(); videoCapture.read(imageBufferMat); videoCapture.set(Videoio.CAP_PROP_POS_FRAMES, 0); videoCapture.grab(); baseMat = new Mat(imageBufferMat.rows(), imageBufferMat.cols(), imageBufferMat.type()); imageBufferMat.copyTo(baseMat); if (baseColorFormat == outputColorFormat) { frameMat = baseMat; } else { frameMat = new Mat(baseMat.rows(), baseMat.cols(), CvType.CV_8UC(Channels(outputColorFormat))); } isPlaying = true; Debug.Log("VideoCaptrueToMatHelper:: " + " filePath:" + requestedVideoFilePath + " width:" + frameMat.width() + " height:" + frameMat.height() + " fps:" + videoCapture.get(Videoio.CAP_PROP_FPS)); isInitWaiting = false; hasInitDone = true; initCoroutine = null; getFilePathCoroutine = null; if (onInitialized != null) onInitialized.Invoke(); StartWaitFrameTimeThread(); break; } else { initFrameCount++; yield return null; } } if (isTimeout) { videoCapture = null; isInitWaiting = false; initCoroutine = null; getFilePathCoroutine = null; if (onErrorOccurred != null) onErrorOccurred.Invoke(ErrorCode.TIMEOUT); } } protected virtual void StartWaitFrameTimeThread() { if (isThreadRunning) return; //Debug.Log("Thread Start"); isThreadRunning = true; shouldStopThread = false; #if UNITY_WEBGL waitFrameTimeCoroutine = WaitFrameTimeThreadWorker(); StartCoroutine(waitFrameTimeCoroutine); #elif WINDOWS_UWP || (!UNITY_WSA_10_0 && (NET_4_6 || NET_STANDARD_2_0)) System.Threading.Tasks.Task.Run(() => WaitFrameTimeThreadWorker()); #else System.Threading.ThreadPool.QueueUserWorkItem(_ => WaitFrameTimeThreadWorker()); #endif } protected virtual void StopWaitFrameTimeThread() { if (!isThreadRunning) return; #if UNITY_WEBGL if (waitFrameTimeCoroutine != null) { StopCoroutine(waitFrameTimeCoroutine); ((IDisposable)waitFrameTimeCoroutine).Dispose(); waitFrameTimeCoroutine = null; } isThreadRunning = false; #else shouldStopThread = true; #endif while (isThreadRunning) { //Wait threading stop } //Debug.Log("Thread Stop"); } #if UNITY_WEBGL protected virtual IEnumerator WaitFrameTimeThreadWorker() { double videoFPS = (videoCapture.get(Videoio.CAP_PROP_FPS) <= 0) ? 10.0 : videoCapture.get(Videoio.CAP_PROP_FPS); float frameTime_sec = (float)(1000.0 / videoFPS / 1000.0); WaitForSeconds wait = new WaitForSeconds(frameTime_sec); while (true) { if (isPlaying) { while (isReading || didUpdateThisFrame) { yield return null; } didUpdateThisFrame = true; didUpdateImageBufferInCurrentFrame = true; yield return wait; } else { yield return null; } } } #else protected virtual void WaitFrameTimeThreadWorker() { double videoFPS = (videoCapture.get(Videoio.CAP_PROP_FPS) <= 0) ? 10.0 : videoCapture.get(Videoio.CAP_PROP_FPS); int frameTime_msec = (int)(1000.0 / videoFPS); while (!shouldStopThread) { if (isPlaying) { while (isReading || didUpdateThisFrame) { } didUpdateThisFrame = true; didUpdateImageBufferInCurrentFrame = true; System.Threading.Thread.Sleep(frameTime_msec); } } isThreadRunning = false; } #endif /// /// Indicates whether this instance has been initialized. /// /// true, if this instance has been initialized, false otherwise. public virtual bool IsInitialized() { return hasInitDone; } /// /// Starts the video. /// public virtual void Play() { if (hasInitDone) { lock (videoCaptureLockObject) { if (_videoCapture.get(Videoio.CAP_PROP_POS_FRAMES) >= _videoCapture.get(Videoio.CAP_PROP_FRAME_COUNT)) { _videoCapture.set(Videoio.CAP_PROP_POS_FRAMES, 0); _videoCapture.read(imageBufferMat); } } isPlaying = true; } } /// /// Pauses the video. /// public virtual void Pause() { if (hasInitDone) isPlaying = false; } /// /// Stops the video. /// public virtual void Stop() { if (hasInitDone) isPlaying = false; } /// /// Indicates whether the video is currently playing. /// /// true, if the video is playing, false otherwise. public virtual bool IsPlaying() { return hasInitDone ? isPlaying : false; } /// /// Returns the video width. /// /// The video width. public virtual int GetWidth() { if (!hasInitDone) return -1; return frameMat.width(); } /// /// Returns the video height. /// /// The video height. public virtual int GetHeight() { if (!hasInitDone) return -1; return frameMat.height(); } /// /// Returns the video framerate. /// /// The video framerate. public virtual float GetFPS() { return hasInitDone ? (float)videoCapture.get(Videoio.CAP_PROP_FPS) : -1f; } /// /// Returns the relative position of the video file: 0=start of the film, 1=end of the film. /// /// The relative position of the video file: 0=start of the film, 1=end of the film. public virtual float GetPosRatio() { if (hasInitDone) { return (float)videoCapture.get(Videoio.CAP_PROP_POS_AVI_RATIO); } else { return 0f; } } /// /// Sets the relative position of the video file: 0=start of the film, 1=end of the film. /// /// The relative position of the video file: 0=start of the film, 1=end of the film. public virtual void SetPosRatio(float ratio) { if (hasInitDone) { lock (videoCaptureLockObject) { bool supported = _videoCapture.set(Videoio.CAP_PROP_POS_AVI_RATIO, ratio); if (!supported) { _videoCapture.set(Videoio.CAP_PROP_POS_FRAMES, (int)(ratio * _videoCapture.get(Videoio.CAP_PROP_FRAME_COUNT))); } if (_videoCapture.get(Videoio.CAP_PROP_POS_FRAMES) >= _videoCapture.get(Videoio.CAP_PROP_FRAME_COUNT)) { if (loop) { _videoCapture.set(Videoio.CAP_PROP_POS_FRAMES, 0); } else { _videoCapture.set(Videoio.CAP_PROP_POS_FRAMES, (int)_videoCapture.get(Videoio.CAP_PROP_FRAME_COUNT) - 1); isPlaying = false; } } _videoCapture.read(imageBufferMat); } } } /// /// Returns the video base color format. /// /// The video base color format. public virtual ColorFormat GetBaseColorFormat() { return baseColorFormat; } /// /// Returns the VideoCapture instance. /// /// The VideoCapture instance. public virtual VideoCapture GetVideoCapture() { return hasInitDone ? videoCapture : null; } /// /// Indicates whether the video buffer of the frame has been updated. /// /// true, if the video buffer has been updated false otherwise. public virtual bool DidUpdateThisFrame() { if (!hasInitDone) return false; return didUpdateThisFrame; } /// /// Gets the mat of the current frame. /// The Mat object's type is 'CV_8UC4' or 'CV_8UC3' or 'CV_8UC1' (ColorFormat is determined by the outputColorFormat setting). /// /// The mat of the current frame. public virtual Mat GetMat() { if (!hasInitDone) { return frameMat; } didUpdateImageBufferInCurrentFrame = false; if (baseColorFormat == outputColorFormat) { imageBufferMat.copyTo(frameMat); } else { Imgproc.cvtColor(imageBufferMat, frameMat, ColorConversionCodes(baseColorFormat, outputColorFormat)); } return frameMat; } protected virtual int Channels(ColorFormat type) { switch (type) { case ColorFormat.GRAY: return 1; case ColorFormat.RGB: case ColorFormat.BGR: return 3; case ColorFormat.RGBA: case ColorFormat.BGRA: return 4; default: return 4; } } protected virtual int ColorConversionCodes(ColorFormat srcType, ColorFormat dstType) { if (srcType == ColorFormat.GRAY) { if (dstType == ColorFormat.RGB) return Imgproc.COLOR_GRAY2RGB; else if (dstType == ColorFormat.BGR) return Imgproc.COLOR_GRAY2BGR; else if (dstType == ColorFormat.RGBA) return Imgproc.COLOR_GRAY2RGBA; else if (dstType == ColorFormat.BGRA) return Imgproc.COLOR_GRAY2BGRA; } else if (srcType == ColorFormat.RGB) { if (dstType == ColorFormat.GRAY) return Imgproc.COLOR_RGB2GRAY; else if (dstType == ColorFormat.BGR) return Imgproc.COLOR_RGB2BGR; else if (dstType == ColorFormat.RGBA) return Imgproc.COLOR_RGB2RGBA; else if (dstType == ColorFormat.BGRA) return Imgproc.COLOR_RGB2BGRA; } else if (srcType == ColorFormat.BGR) { if (dstType == ColorFormat.GRAY) return Imgproc.COLOR_BGR2GRAY; else if (dstType == ColorFormat.RGB) return Imgproc.COLOR_BGR2RGB; else if (dstType == ColorFormat.RGBA) return Imgproc.COLOR_BGR2RGBA; else if (dstType == ColorFormat.BGRA) return Imgproc.COLOR_BGR2BGRA; } else if (srcType == ColorFormat.RGBA) { if (dstType == ColorFormat.GRAY) return Imgproc.COLOR_RGBA2GRAY; else if (dstType == ColorFormat.RGB) return Imgproc.COLOR_RGBA2RGB; else if (dstType == ColorFormat.BGR) return Imgproc.COLOR_RGBA2BGR; else if (dstType == ColorFormat.BGRA) return Imgproc.COLOR_RGBA2BGRA; } else if (srcType == ColorFormat.BGRA) { if (dstType == ColorFormat.GRAY) return Imgproc.COLOR_BGRA2GRAY; else if (dstType == ColorFormat.RGB) return Imgproc.COLOR_BGRA2RGB; else if (dstType == ColorFormat.BGR) return Imgproc.COLOR_BGRA2BGR; else if (dstType == ColorFormat.RGBA) return Imgproc.COLOR_BGRA2RGBA; } return -1; } /// /// Cancel Init Coroutine. /// protected virtual void CancelInitCoroutine() { if (getFilePathCoroutine != null) { StopCoroutine(getFilePathCoroutine); ((IDisposable)getFilePathCoroutine).Dispose(); getFilePathCoroutine = null; } if (initCoroutine != null) { StopCoroutine(initCoroutine); ((IDisposable)initCoroutine).Dispose(); initCoroutine = null; } } /// /// To release the resources. /// protected virtual void ReleaseResources() { isInitWaiting = false; hasInitDone = false; isReading = false; isPlaying = false; didUpdateThisFrame = false; didUpdateImageBufferInCurrentFrame = false; if (videoCapture != null) { videoCapture.Dispose(); videoCapture = null; } if (frameMat != null) { frameMat.Dispose(); frameMat = null; } if (baseMat != null) { baseMat.Dispose(); baseMat = null; } if (imageBufferMat != null) { imageBufferMat.Dispose(); imageBufferMat = null; } StopWaitFrameTimeThread(); } /// /// Releases all resource used by the object. /// /// Call when you are finished using the . The /// method leaves the in an unusable state. After /// calling , you must release all references to the so /// the garbage collector can reclaim the memory that the was occupying. public virtual void Dispose() { if (isInitWaiting) { CancelInitCoroutine(); ReleaseResources(); } else if (hasInitDone) { ReleaseResources(); if (onDisposed != null) onDisposed.Invoke(); } } } }