ScCN/scratch-gui
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move runtime/renderer related code back into the vm io device. Refactor ModalVideoManager to use the video provider.

Browse Source
This commit is contained in:
Karishma Chadha committed 2018-04-30 17:18:24 -04:00
1 parent a7a30929b4
commit 5d87872361
3 files changed
+53 -297
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/containers/camera-modal.jsx
+2 -2
View File
@@ -4,7 +4,7 @@ import React from 'react';
import {connect} from 'react-redux';
import CameraModalComponent from '../components/camera-modal/camera-modal.jsx';
import {ModalVideoProvider} from '../lib/camera.js';
import {ModalVideoManager} from '../lib/camera.js';
import {
closeCameraCapture
@@ -46,7 +46,7 @@ class CameraModal extends React.Component {
setCanvas (canvas) {
this.canvas = canvas;
if (this.canvas) {
this.videoDevice = new ModalVideoProvider(this.canvas);
this.videoDevice = new ModalVideoManager(this.canvas);
this.videoDevice.enableVideo();
}
}
src/containers/stage.jsx
+2 -2
View File
@@ -6,7 +6,7 @@ import VM from 'scratch-vm';
import {connect} from 'react-redux';
import {getEventXY} from '../lib/touch-utils';
import {stageVideoProvider} from '../lib/camera';
import {VideoProvider} from '../lib/camera';
import StageComponent from '../components/stage/stage.jsx';
@@ -58,7 +58,7 @@ class Stage extends React.Component {
this.renderer = new Renderer(this.canvas);
this.props.vm.attachRenderer(this.renderer);
this.props.vm.runtime.addListener('QUESTION', this.questionListener);
this.props.vm.setVideoProvider(stageVideoProvider(this.props.vm.runtime));
this.props.vm.setVideoProvider(new VideoProvider());
}
shouldComponentUpdate (nextProps, nextState) {
return this.props.width !== nextProps.width ||
src/lib/camera.js
+49 -293
View File
@@ -1,8 +1,14 @@
import getUserMedia from 'get-user-media-promise';
import log from './log.js';
// Single Setup For All Video Streams used by the GUI
// While VideoProvider uses a private _singleSetup
// property to ensure that each instance of a VideoProvider
// use the same setup, this ensures that all instances
// of VideoProviders use a single stream. This way, closing a camera modal
// does not affect the video on the stage, and a program running and disabling
// video on the stage will not affect the camera modal's video.
const requestStack = [];
// Single Setup For All Video Streams
const requestVideoStream = videoDesc => {
let streamPromise;
if (requestStack.length === 0) {
@@ -24,14 +30,11 @@ const requestDisableCheck = () => {
return true;
};
class StageVideoProvider {
constructor (runtime) {
/**
* Reference to the owning Runtime.
* @type{!Runtime}
*/
this.runtime = runtime;
/**
* Video Manager for video extensions.
*/
class VideoProvider {
constructor () {
/**
* Default value for mirrored frames.
* @type boolean
@@ -60,32 +63,6 @@ class StageVideoProvider {
* Stores some canvas/frame data per resolution/mirror states
*/
this._workspace = [];
/**
* Id representing a Scratch Renderer skin the video is rendered to for
* previewing.
* @type {number}
*/
this._skinId = -1;
/**
* The Scratch Renderer Skin object.
* @type {Skin}
*/
this._skin = null;
/**
* Id for a drawable using the video's skin that will render as a video
* preview.
* @type {Drawable}
*/
this._drawable = -1;
/**
* Store the last state of the video transparency ghost effect
* @type {number}
*/
this._ghost = 0;
}
static get FORMAT_IMAGE_DATA () {
@@ -113,6 +90,13 @@ class StageVideoProvider {
return 1;
}
/**
* Get the HTML video element containing the stream
*/
get video () {
return this._video;
}
/**
* Request video be enabled. Sets up video, creates video skin and enables preview.
*
@@ -146,7 +130,6 @@ class StageVideoProvider {
// we might be asked to re-enable before _teardown is called, just ignore it.
if (this.enabled === false) {
const disableTrack = requestDisableCheck();
this._disablePreview();
this._singleSetup = null;
// by clearing refs to video and track, we should lose our hold over the camera
this._video = null;
@@ -171,9 +154,9 @@ class StageVideoProvider {
* @return {ArrayBuffer|Canvas|string|null} Frame data in requested format, null when errors.
*/
getFrame ({
dimensions = StageVideoProvider.DIMENSIONS,
dimensions = VideoProvider.DIMENSIONS,
mirror = this.mirror,
format = StageVideoProvider.FORMAT_IMAGE_DATA,
format = VideoProvider.FORMAT_IMAGE_DATA,
cacheTimeout = this._frameCacheTimeout
}) {
if (!this.videoReady) {
@@ -211,9 +194,9 @@ class StageVideoProvider {
const formatCache = cacheData[format];
if (formatCache.lastUpdate + cacheTimeout < now) {
if (format === StageVideoProvider.FORMAT_IMAGE_DATA) {
if (format === VideoProvider.FORMAT_IMAGE_DATA) {
formatCache.lastData = context.getImageData(0, 0, width, height);
} else if (format === StageVideoProvider.FORMAT_CANVAS) {
} else if (format === VideoProvider.FORMAT_CANVAS) {
// this will never change
formatCache.lastUpdate = Infinity;
formatCache.lastData = canvas;
@@ -231,17 +214,6 @@ class StageVideoProvider {
return formatCache.lastData;
}
/**
* Set the preview ghost effect
* @param {number} ghost from 0 (visible) to 100 (invisible) - ghost effect
*/
setPreviewGhost (ghost) {
this._ghost = ghost;
if (this._drawable) {
this.runtime.renderer.updateDrawableProperties(this._drawable, {ghost});
}
}
/**
* Method called when an error happens. Default implementation is just to log error.
*
@@ -265,7 +237,7 @@ class StageVideoProvider {
return this._singleSetup;
}
this._singleSetup = requestVideoStream({ // navigator.mediaDevices.getUserMedia({
this._singleSetup = requestVideoStream({
width: {min: 480, ideal: 640},
height: {min: 360, ideal: 480}
})
@@ -284,7 +256,6 @@ class StageVideoProvider {
// needed.
this._video.play(); // Needed for Safari/Firefox, Chrome auto-plays.
this._track = stream.getTracks()[0];
this._setupPreview();
return this;
})
.catch(error => {
@@ -295,63 +266,6 @@ class StageVideoProvider {
return this._singleSetup;
}
_disablePreview () {
if (this._skin) {
this._skin.clear();
this.runtime.renderer.updateDrawableProperties(this._drawable, {visible: false});
}
this._renderPreviewFrame = null;
}
_setupPreview () {
const {renderer} = this.runtime;
if (!renderer) return;
if (this._skinId === -1 && this._skin === null && this._drawable === -1) {
this._skinId = renderer.createPenSkin();
this._skin = renderer._allSkins[this._skinId];
this._drawable = renderer.createDrawable();
renderer.setDrawableOrder(
this._drawable,
StageVideoProvider.ORDER
);
renderer.updateDrawableProperties(this._drawable, {
skinId: this._skinId
});
}
// if we haven't already created and started a preview frame render loop, do so
if (!this._renderPreviewFrame) {
renderer.updateDrawableProperties(this._drawable, {
ghost: this._ghost,
visible: true
});
this._renderPreviewFrame = () => {
clearTimeout(this._renderPreviewTimeout);
if (!this._renderPreviewFrame) {
return;
}
this._renderPreviewTimeout = setTimeout(this._renderPreviewFrame, this.runtime.currentStepTime);
const canvas = this.getFrame({format: StageVideoProvider.FORMAT_CANVAS});
if (!canvas) {
this._skin.clear();
return;
}
const xOffset = StageVideoProvider.DIMENSIONS[0] / -2;
const yOffset = StageVideoProvider.DIMENSIONS[1] / 2;
this._skin.drawStamp(canvas, xOffset, yOffset);
this.runtime.requestRedraw();
};
this._renderPreviewFrame();
}
}
get videoReady () {
if (!this.enabled) {
return false;
@@ -401,90 +315,46 @@ class StageVideoProvider {
}
}
class ModalVideoProvider {
/**
* Video Manager for Camera Modal
*/
class ModalVideoManager {
constructor (canvas) {
this._videoProvider = new VideoProvider();
/**
* Captured image data
*/
/**
* Cache frames for this many ms.
* Frame update interval
* @type number
*/
this._frameCacheTimeout = 16;
this._frameTimeout = 16;
this._canvas = canvas;
// These values are double the stage dimensions so that the resulting
// image does not have to get sized down to accomodate double resolution
this._canvasWidth = 960; // Double Stage Width
this._canvasHeight = 720; // Double Stage Height
this._video = null;
// this._workspace = [];
/**
* Usermedia stream track
* @private
*/
this._track = null;
}
/**
* Dimensions the video stream is analyzed at after its rendered to the
* sample canvas.
* @type {Array.<number>}
*/
static get DIMENSIONS () {
return [480, 360];
}
setVideo (video) {
this._video = video;
}
enableVideo () {
const thisContext = this;
this._video = this._video ? this._video : document.createElement('video');
// TODO possibly make common function for this
requestVideoStream(true)
.then(userMediaStream => {
try {
thisContext._video.srcObject = userMediaStream;
} catch (e) {
thisContext._video.src = window.URL.createObjectURL(userMediaStream);
}
thisContext._track = userMediaStream.getTracks()[0];
this._videoProvider.enableVideo().then(() => {
thisContext._width = 960; // abstract this out
thisContext._height = 720;
const ctx = this._canvas.getContext('2d');
ctx.scale(-1, 1);
ctx.translate(this._canvasWidth * -1, 0);
const ctx = this._canvas.getContext('2d');
ctx.scale(-1, 1);
ctx.translate(thisContext._width * -1, 0);
thisContext._drawFrames();
// The following also works...
// thisContext._videoFeedInterval = setInterval(
// () => this.getFrame({format: ModalVideoProvider.FORMAT_CANVAS}),
// thisContext._frameCacheTimeout);
// thisContext._setupPreview();
})
.catch(e => {
log.warn(e);
});
this._drawFrames();
});
}
_drawFrames () {
const video = this._videoProvider.video;
this._videoFeedInterval = setInterval(() =>
this._canvas.getContext('2d').drawImage(this._video,
this._canvas.getContext('2d').drawImage(video,
// source x, y, width, height
0, 0, this._video.videoWidth, this._video.videoHeight,
0, 0, video.videoWidth, video.videoHeight,
// dest x, y, width, height
0, 0, this._width, this._height
), this._frameCacheTimeout);
0, 0, this._canvasWidth, this._canvasHeight
), this._frameTimeout);
}
takeSnapshot () {
@@ -496,126 +366,12 @@ class ModalVideoProvider {
this._drawFrames();
}
/**
* get an internal workspace for canvas/context/caches
* this uses some document stuff to create a canvas and what not, probably needs abstraction
* into the renderer layer?
* @private
* @return {object} A workspace for canvas/data storage. Internal format not documented intentionally
*/
// _getWorkspace ({dimensions, mirror}) {
// let workspace = this._workspace.find(space => (
// space.dimensions.join('-') === dimensions.join('-') &&
// space.mirror === mirror
// ));
// if (!workspace) {
// workspace = {
// dimensions,
// mirror,
// canvas: this._canvas, // document.createElement('canvas'),
// lastUpdate: 0,
// cacheData: {}
// };
// workspace.canvas.width = dimensions[0];
// workspace.canvas.height = dimensions[1];
// workspace.context = workspace.canvas.getContext('2d');
// this._workspace.push(workspace);
// }
// return workspace;
// }
/**
* Return frame data from the video feed in a specified dimensions, format, and mirroring.
*
* @param {object} frameInfo A descriptor of the frame you would like to receive.
* @param {Array.<number>} frameInfo.dimensions [width, height] array of numbers. Defaults to [480,360]
* @param {boolean} frameInfo.mirror If you specificly want a mirror/non-mirror frame, defaults to the global
* mirror state (ioDevices.video.mirror)
* @param {string} frameInfo.format Requested video format, available formats are 'image-data' and 'canvas'.
* @param {number} frameInfo.cacheTimeout Will reuse previous image data if the time since capture is less than
* the cacheTimeout. Defaults to 16ms.
*
* @return {ArrayBuffer|Canvas|string|null} Frame data in requested format, null when errors.
*/
// getFrame ({
// dimensions = ModalVideoProvider.DIMENSIONS,
// mirror = this.mirror,
// format = ModalVideoProvider.FORMAT_IMAGE_DATA,
// cacheTimeout = this._frameCacheTimeout
// }) {
// if (!this.videoReady) {
// return null;
// }
// const [width, height] = dimensions;
// const workspace = this._getWorkspace({dimensions, mirror: Boolean(mirror)});
// const {videoWidth, videoHeight} = this._video;
// const {canvas, context, lastUpdate, cacheData} = workspace;
// const now = Date.now();
//
// // if the canvas hasn't been updated...
// if (lastUpdate + cacheTimeout < now) {
//
// if (mirror) {
// context.scale(-1, 1);
// context.translate(width * -1, 0);
// }
//
// context.drawImage(this._video,
// // source x, y, width, height
// 0, 0, videoWidth, videoHeight,
// // dest x, y, width, height
// 0, 0, width, height
// );
//
// context.resetTransform();
// workspace.lastUpdate = now;
// }
//
// // each data type has it's own data cache, but the canvas is the same
// if (!cacheData[format]) {
// cacheData[format] = {lastUpdate: 0};
// }
// const formatCache = cacheData[format];
//
// if (formatCache.lastUpdate + cacheTimeout < now) {
// if (format === ModalVideoProvider.FORMAT_IMAGE_DATA) {
// formatCache.lastData = context.getImageData(0, 0, width, height);
// } else if (format === ModalVideoProvider.FORMAT_CANVAS) {
// // this will never change
// formatCache.lastUpdate = Infinity;
// formatCache.lastData = canvas;
// } else {
// log.error(`video io error - unimplemented format ${format}`);
// // cache the null result forever, don't log about it again..
// formatCache.lastUpdate = Infinity;
// formatCache.lastData = null;
// }
//
// // rather than set to now, this data is as stale as it's canvas is
// formatCache.lastUpdate = Math.max(workspace.lastUpdate, formatCache.lastUpdate);
// }
//
// return formatCache.lastData;
// }
disableVideo () {
// Don't need to do anything with this check,
// but we do want to pop our use of the stream off the stack
const disableTrack = requestDisableCheck();
if (this._video) {
if (this._track && disableTrack) this._track.stop();
this._video.pause();
this._video.srcObject = null;
}
this._videoProvider.disableVideo();
}
}
const stageVideoProvider = function (runtime) {
return new StageVideoProvider(runtime);
};
export {
stageVideoProvider,
ModalVideoProvider
VideoProvider,
ModalVideoManager
};