diff --git a/src/containers/camera-modal.jsx b/src/containers/camera-modal.jsx index e5cf3fe59..cfc4284aa 100644 --- a/src/containers/camera-modal.jsx +++ b/src/containers/camera-modal.jsx @@ -4,7 +4,7 @@ import React from 'react'; import {connect} from 'react-redux'; import CameraModalComponent from '../components/camera-modal/camera-modal.jsx'; -import {ModalVideoProvider} from '../lib/camera.js'; +import {ModalVideoManager} from '../lib/camera.js'; import { closeCameraCapture @@ -46,7 +46,7 @@ class CameraModal extends React.Component { setCanvas (canvas) { this.canvas = canvas; if (this.canvas) { - this.videoDevice = new ModalVideoProvider(this.canvas); + this.videoDevice = new ModalVideoManager(this.canvas); this.videoDevice.enableVideo(); } } diff --git a/src/containers/stage.jsx b/src/containers/stage.jsx index e6918fdd2..7e543617a 100644 --- a/src/containers/stage.jsx +++ b/src/containers/stage.jsx @@ -6,7 +6,7 @@ import VM from 'scratch-vm'; import {connect} from 'react-redux'; import {getEventXY} from '../lib/touch-utils'; -import {stageVideoProvider} from '../lib/camera'; +import {VideoProvider} from '../lib/camera'; import StageComponent from '../components/stage/stage.jsx'; @@ -58,7 +58,7 @@ class Stage extends React.Component { this.renderer = new Renderer(this.canvas); this.props.vm.attachRenderer(this.renderer); this.props.vm.runtime.addListener('QUESTION', this.questionListener); - this.props.vm.setVideoProvider(stageVideoProvider(this.props.vm.runtime)); + this.props.vm.setVideoProvider(new VideoProvider()); } shouldComponentUpdate (nextProps, nextState) { return this.props.width !== nextProps.width || diff --git a/src/lib/camera.js b/src/lib/camera.js index 4b6a08e7a..339d59dc2 100644 --- a/src/lib/camera.js +++ b/src/lib/camera.js @@ -1,8 +1,14 @@ import getUserMedia from 'get-user-media-promise'; import log from './log.js'; +// Single Setup For All Video Streams used by the GUI +// While VideoProvider uses a private _singleSetup +// property to ensure that each instance of a VideoProvider +// use the same setup, this ensures that all instances +// of VideoProviders use a single stream. This way, closing a camera modal +// does not affect the video on the stage, and a program running and disabling +// video on the stage will not affect the camera modal's video. const requestStack = []; -// Single Setup For All Video Streams const requestVideoStream = videoDesc => { let streamPromise; if (requestStack.length === 0) { @@ -24,14 +30,11 @@ const requestDisableCheck = () => { return true; }; -class StageVideoProvider { - constructor (runtime) { - /** - * Reference to the owning Runtime. - * @type{!Runtime} - */ - this.runtime = runtime; - +/** + * Video Manager for video extensions. + */ +class VideoProvider { + constructor () { /** * Default value for mirrored frames. * @type boolean @@ -60,32 +63,6 @@ class StageVideoProvider { * Stores some canvas/frame data per resolution/mirror states */ this._workspace = []; - - /** - * Id representing a Scratch Renderer skin the video is rendered to for - * previewing. - * @type {number} - */ - this._skinId = -1; - - /** - * The Scratch Renderer Skin object. - * @type {Skin} - */ - this._skin = null; - - /** - * Id for a drawable using the video's skin that will render as a video - * preview. - * @type {Drawable} - */ - this._drawable = -1; - - /** - * Store the last state of the video transparency ghost effect - * @type {number} - */ - this._ghost = 0; } static get FORMAT_IMAGE_DATA () { @@ -113,6 +90,13 @@ class StageVideoProvider { return 1; } + /** + * Get the HTML video element containing the stream + */ + get video () { + return this._video; + } + /** * Request video be enabled. Sets up video, creates video skin and enables preview. * @@ -146,7 +130,6 @@ class StageVideoProvider { // we might be asked to re-enable before _teardown is called, just ignore it. if (this.enabled === false) { const disableTrack = requestDisableCheck(); - this._disablePreview(); this._singleSetup = null; // by clearing refs to video and track, we should lose our hold over the camera this._video = null; @@ -171,9 +154,9 @@ class StageVideoProvider { * @return {ArrayBuffer|Canvas|string|null} Frame data in requested format, null when errors. */ getFrame ({ - dimensions = StageVideoProvider.DIMENSIONS, + dimensions = VideoProvider.DIMENSIONS, mirror = this.mirror, - format = StageVideoProvider.FORMAT_IMAGE_DATA, + format = VideoProvider.FORMAT_IMAGE_DATA, cacheTimeout = this._frameCacheTimeout }) { if (!this.videoReady) { @@ -211,9 +194,9 @@ class StageVideoProvider { const formatCache = cacheData[format]; if (formatCache.lastUpdate + cacheTimeout < now) { - if (format === StageVideoProvider.FORMAT_IMAGE_DATA) { + if (format === VideoProvider.FORMAT_IMAGE_DATA) { formatCache.lastData = context.getImageData(0, 0, width, height); - } else if (format === StageVideoProvider.FORMAT_CANVAS) { + } else if (format === VideoProvider.FORMAT_CANVAS) { // this will never change formatCache.lastUpdate = Infinity; formatCache.lastData = canvas; @@ -231,17 +214,6 @@ class StageVideoProvider { return formatCache.lastData; } - /** - * Set the preview ghost effect - * @param {number} ghost from 0 (visible) to 100 (invisible) - ghost effect - */ - setPreviewGhost (ghost) { - this._ghost = ghost; - if (this._drawable) { - this.runtime.renderer.updateDrawableProperties(this._drawable, {ghost}); - } - } - /** * Method called when an error happens. Default implementation is just to log error. * @@ -265,7 +237,7 @@ class StageVideoProvider { return this._singleSetup; } - this._singleSetup = requestVideoStream({ // navigator.mediaDevices.getUserMedia({ + this._singleSetup = requestVideoStream({ width: {min: 480, ideal: 640}, height: {min: 360, ideal: 480} }) @@ -284,7 +256,6 @@ class StageVideoProvider { // needed. this._video.play(); // Needed for Safari/Firefox, Chrome auto-plays. this._track = stream.getTracks()[0]; - this._setupPreview(); return this; }) .catch(error => { @@ -295,63 +266,6 @@ class StageVideoProvider { return this._singleSetup; } - _disablePreview () { - if (this._skin) { - this._skin.clear(); - this.runtime.renderer.updateDrawableProperties(this._drawable, {visible: false}); - } - this._renderPreviewFrame = null; - } - - _setupPreview () { - const {renderer} = this.runtime; - if (!renderer) return; - - if (this._skinId === -1 && this._skin === null && this._drawable === -1) { - this._skinId = renderer.createPenSkin(); - this._skin = renderer._allSkins[this._skinId]; - this._drawable = renderer.createDrawable(); - renderer.setDrawableOrder( - this._drawable, - StageVideoProvider.ORDER - ); - renderer.updateDrawableProperties(this._drawable, { - skinId: this._skinId - }); - } - - // if we haven't already created and started a preview frame render loop, do so - if (!this._renderPreviewFrame) { - renderer.updateDrawableProperties(this._drawable, { - ghost: this._ghost, - visible: true - }); - - this._renderPreviewFrame = () => { - clearTimeout(this._renderPreviewTimeout); - if (!this._renderPreviewFrame) { - return; - } - - this._renderPreviewTimeout = setTimeout(this._renderPreviewFrame, this.runtime.currentStepTime); - - const canvas = this.getFrame({format: StageVideoProvider.FORMAT_CANVAS}); - - if (!canvas) { - this._skin.clear(); - return; - } - - const xOffset = StageVideoProvider.DIMENSIONS[0] / -2; - const yOffset = StageVideoProvider.DIMENSIONS[1] / 2; - this._skin.drawStamp(canvas, xOffset, yOffset); - this.runtime.requestRedraw(); - }; - - this._renderPreviewFrame(); - } - } - get videoReady () { if (!this.enabled) { return false; @@ -401,90 +315,46 @@ class StageVideoProvider { } } -class ModalVideoProvider { +/** + * Video Manager for Camera Modal + */ +class ModalVideoManager { constructor (canvas) { + this._videoProvider = new VideoProvider(); /** - * Captured image data - */ - - /** - * Cache frames for this many ms. + * Frame update interval * @type number */ - this._frameCacheTimeout = 16; + this._frameTimeout = 16; this._canvas = canvas; + // These values are double the stage dimensions so that the resulting + // image does not have to get sized down to accomodate double resolution + this._canvasWidth = 960; // Double Stage Width + this._canvasHeight = 720; // Double Stage Height - this._video = null; - - // this._workspace = []; - - /** - * Usermedia stream track - * @private - */ - this._track = null; } - - /** - * Dimensions the video stream is analyzed at after its rendered to the - * sample canvas. - * @type {Array.} - */ - static get DIMENSIONS () { - return [480, 360]; - } - - setVideo (video) { - this._video = video; - } - - enableVideo () { - const thisContext = this; - this._video = this._video ? this._video : document.createElement('video'); - // TODO possibly make common function for this - requestVideoStream(true) - .then(userMediaStream => { - try { - thisContext._video.srcObject = userMediaStream; - } catch (e) { - thisContext._video.src = window.URL.createObjectURL(userMediaStream); - } - thisContext._track = userMediaStream.getTracks()[0]; + this._videoProvider.enableVideo().then(() => { - thisContext._width = 960; // abstract this out - thisContext._height = 720; + const ctx = this._canvas.getContext('2d'); + ctx.scale(-1, 1); + ctx.translate(this._canvasWidth * -1, 0); - const ctx = this._canvas.getContext('2d'); - - ctx.scale(-1, 1); - ctx.translate(thisContext._width * -1, 0); - - thisContext._drawFrames(); - - // The following also works... - // thisContext._videoFeedInterval = setInterval( - // () => this.getFrame({format: ModalVideoProvider.FORMAT_CANVAS}), - // thisContext._frameCacheTimeout); - - - // thisContext._setupPreview(); - }) - .catch(e => { - log.warn(e); - }); + this._drawFrames(); + }); } _drawFrames () { + const video = this._videoProvider.video; this._videoFeedInterval = setInterval(() => - this._canvas.getContext('2d').drawImage(this._video, + this._canvas.getContext('2d').drawImage(video, // source x, y, width, height - 0, 0, this._video.videoWidth, this._video.videoHeight, + 0, 0, video.videoWidth, video.videoHeight, // dest x, y, width, height - 0, 0, this._width, this._height - ), this._frameCacheTimeout); + 0, 0, this._canvasWidth, this._canvasHeight + ), this._frameTimeout); } takeSnapshot () { @@ -496,126 +366,12 @@ class ModalVideoProvider { this._drawFrames(); } - /** - * get an internal workspace for canvas/context/caches - * this uses some document stuff to create a canvas and what not, probably needs abstraction - * into the renderer layer? - * @private - * @return {object} A workspace for canvas/data storage. Internal format not documented intentionally - */ - // _getWorkspace ({dimensions, mirror}) { - // let workspace = this._workspace.find(space => ( - // space.dimensions.join('-') === dimensions.join('-') && - // space.mirror === mirror - // )); - // if (!workspace) { - // workspace = { - // dimensions, - // mirror, - // canvas: this._canvas, // document.createElement('canvas'), - // lastUpdate: 0, - // cacheData: {} - // }; - // workspace.canvas.width = dimensions[0]; - // workspace.canvas.height = dimensions[1]; - // workspace.context = workspace.canvas.getContext('2d'); - // this._workspace.push(workspace); - // } - // return workspace; - // } - - /** - * Return frame data from the video feed in a specified dimensions, format, and mirroring. - * - * @param {object} frameInfo A descriptor of the frame you would like to receive. - * @param {Array.} frameInfo.dimensions [width, height] array of numbers. Defaults to [480,360] - * @param {boolean} frameInfo.mirror If you specificly want a mirror/non-mirror frame, defaults to the global - * mirror state (ioDevices.video.mirror) - * @param {string} frameInfo.format Requested video format, available formats are 'image-data' and 'canvas'. - * @param {number} frameInfo.cacheTimeout Will reuse previous image data if the time since capture is less than - * the cacheTimeout. Defaults to 16ms. - * - * @return {ArrayBuffer|Canvas|string|null} Frame data in requested format, null when errors. - */ - // getFrame ({ - // dimensions = ModalVideoProvider.DIMENSIONS, - // mirror = this.mirror, - // format = ModalVideoProvider.FORMAT_IMAGE_DATA, - // cacheTimeout = this._frameCacheTimeout - // }) { - // if (!this.videoReady) { - // return null; - // } - // const [width, height] = dimensions; - // const workspace = this._getWorkspace({dimensions, mirror: Boolean(mirror)}); - // const {videoWidth, videoHeight} = this._video; - // const {canvas, context, lastUpdate, cacheData} = workspace; - // const now = Date.now(); - // - // // if the canvas hasn't been updated... - // if (lastUpdate + cacheTimeout < now) { - // - // if (mirror) { - // context.scale(-1, 1); - // context.translate(width * -1, 0); - // } - // - // context.drawImage(this._video, - // // source x, y, width, height - // 0, 0, videoWidth, videoHeight, - // // dest x, y, width, height - // 0, 0, width, height - // ); - // - // context.resetTransform(); - // workspace.lastUpdate = now; - // } - // - // // each data type has it's own data cache, but the canvas is the same - // if (!cacheData[format]) { - // cacheData[format] = {lastUpdate: 0}; - // } - // const formatCache = cacheData[format]; - // - // if (formatCache.lastUpdate + cacheTimeout < now) { - // if (format === ModalVideoProvider.FORMAT_IMAGE_DATA) { - // formatCache.lastData = context.getImageData(0, 0, width, height); - // } else if (format === ModalVideoProvider.FORMAT_CANVAS) { - // // this will never change - // formatCache.lastUpdate = Infinity; - // formatCache.lastData = canvas; - // } else { - // log.error(`video io error - unimplemented format ${format}`); - // // cache the null result forever, don't log about it again.. - // formatCache.lastUpdate = Infinity; - // formatCache.lastData = null; - // } - // - // // rather than set to now, this data is as stale as it's canvas is - // formatCache.lastUpdate = Math.max(workspace.lastUpdate, formatCache.lastUpdate); - // } - // - // return formatCache.lastData; - // } - disableVideo () { - // Don't need to do anything with this check, - // but we do want to pop our use of the stream off the stack - const disableTrack = requestDisableCheck(); - - if (this._video) { - if (this._track && disableTrack) this._track.stop(); - this._video.pause(); - this._video.srcObject = null; - } + this._videoProvider.disableVideo(); } } -const stageVideoProvider = function (runtime) { - return new StageVideoProvider(runtime); -}; - export { - stageVideoProvider, - ModalVideoProvider + VideoProvider, + ModalVideoManager };