5 files changed
+387
-1
No files matched your search
+1
-1
@@ -95,7 +95,7 @@
|
||||
"scratch-render": "0.1.0-prerelease.20180427143155",
|
||||
"scratch-svg-renderer": "0.1.0-prerelease.20180423193917",
|
||||
"scratch-storage": "0.4.1",
|
||||
"scratch-vm": "0.1.0-prerelease.1524765647",
|
||||
"scratch-vm": "0.1.0-prerelease.1525095619",
|
||||
"selenium-webdriver": "3.6.0",
|
||||
"startaudiocontext": "1.2.1",
|
||||
"style-loader": "^0.20.0",
|
||||
|
||||
@@ -6,6 +6,7 @@ import VM from 'scratch-vm';
|
||||
import {connect} from 'react-redux';
|
||||
|
||||
import {getEventXY} from '../lib/touch-utils';
|
||||
import VideoProvider from '../lib/video/video-provider';
|
||||
|
||||
import StageComponent from '../components/stage/stage.jsx';
|
||||
|
||||
@@ -57,6 +58,7 @@ class Stage extends React.Component {
|
||||
this.renderer = new Renderer(this.canvas);
|
||||
this.props.vm.attachRenderer(this.renderer);
|
||||
this.props.vm.runtime.addListener('QUESTION', this.questionListener);
|
||||
this.props.vm.setVideoProvider(new VideoProvider());
|
||||
}
|
||||
shouldComponentUpdate (nextProps, nextState) {
|
||||
return this.props.width !== nextProps.width ||
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import getUserMedia from 'get-user-media-promise';
|
||||
|
||||
// Single Setup For All Video Streams used by the GUI
|
||||
// While VideoProvider uses a private _singleSetup
|
||||
// property to ensure that each instance of a VideoProvider
|
||||
// use the same setup, this ensures that all instances
|
||||
// of VideoProviders use a single stream. This way, closing a camera modal
|
||||
// does not affect the video on the stage, and a program running and disabling
|
||||
// video on the stage will not affect the camera modal's video.
|
||||
const requestStack = [];
|
||||
const requestVideoStream = videoDesc => {
|
||||
let streamPromise;
|
||||
if (requestStack.length === 0) {
|
||||
streamPromise = getUserMedia({
|
||||
audio: false,
|
||||
video: videoDesc
|
||||
});
|
||||
requestStack.push(streamPromise);
|
||||
} else if (requestStack.length > 0) {
|
||||
streamPromise = requestStack[0];
|
||||
requestStack.push(true);
|
||||
}
|
||||
return streamPromise;
|
||||
};
|
||||
|
||||
const requestDisableVideo = () => {
|
||||
requestStack.pop();
|
||||
if (requestStack.length > 0) return false;
|
||||
return true;
|
||||
};
|
||||
|
||||
export {
|
||||
requestVideoStream,
|
||||
requestDisableVideo
|
||||
};
|
||||
@@ -0,0 +1,62 @@
|
||||
import VideoProvider from './video-provider.js';
|
||||
/**
|
||||
* Video Manager for Camera Modal
|
||||
*/
|
||||
class ModalVideoManager {
|
||||
constructor (canvas) {
|
||||
this._videoProvider = new VideoProvider();
|
||||
|
||||
this._frameTimeout = 16;
|
||||
|
||||
this._canvas = canvas;
|
||||
// These values are double the stage dimensions so that the resulting
|
||||
// image does not have to get sized down to accomodate double resolution
|
||||
this._canvasWidth = 960; // Double Stage Width
|
||||
this._canvasHeight = 720; // Double Stage Height
|
||||
|
||||
}
|
||||
|
||||
enableVideo (onPermissionSuccess, onVideoLoaded) {
|
||||
const thisContext = this;
|
||||
this._videoProvider.enableVideo(onVideoLoaded).then(() => {
|
||||
if (onPermissionSuccess) onPermissionSuccess();
|
||||
const ctx = thisContext._canvas.getContext('2d');
|
||||
ctx.scale(-1, 1);
|
||||
ctx.translate(thisContext._canvasWidth * -1, 0);
|
||||
|
||||
if (onVideoLoaded) {
|
||||
thisContext._videoProvider.video.onloadeddata = () => {
|
||||
onVideoLoaded();
|
||||
};
|
||||
}
|
||||
|
||||
thisContext._drawFrames();
|
||||
});
|
||||
}
|
||||
|
||||
_drawFrames () {
|
||||
const video = this._videoProvider.video;
|
||||
this._videoFeedInterval = setInterval(() =>
|
||||
this._canvas.getContext('2d').drawImage(video,
|
||||
// source x, y, width, height
|
||||
0, 0, video.videoWidth, video.videoHeight,
|
||||
// dest x, y, width, height
|
||||
0, 0, this._canvasWidth, this._canvasHeight
|
||||
), this._frameTimeout);
|
||||
}
|
||||
|
||||
takeSnapshot () {
|
||||
clearInterval(this._videoFeedInterval);
|
||||
return this._canvas.toDataURL('image/png');
|
||||
}
|
||||
|
||||
clearSnapshot () {
|
||||
this._drawFrames();
|
||||
}
|
||||
|
||||
disableVideo () {
|
||||
this._videoProvider.disableVideo();
|
||||
}
|
||||
}
|
||||
|
||||
export default ModalVideoManager;
|
||||
@@ -0,0 +1,287 @@
|
||||
import {requestVideoStream, requestDisableVideo} from './camera.js';
|
||||
import log from '../log.js';
|
||||
|
||||
/**
|
||||
* Video Manager for video extensions.
|
||||
*/
|
||||
class VideoProvider {
|
||||
constructor () {
|
||||
/**
|
||||
* Default value for mirrored frames.
|
||||
* @type boolean
|
||||
*/
|
||||
this.mirror = true;
|
||||
|
||||
/**
|
||||
* Cache frames for this many ms.
|
||||
* @type number
|
||||
*/
|
||||
this._frameCacheTimeout = 16;
|
||||
|
||||
/**
|
||||
* DOM Video element
|
||||
* @private
|
||||
*/
|
||||
this._video = null;
|
||||
|
||||
/**
|
||||
* Usermedia stream track
|
||||
* @private
|
||||
*/
|
||||
this._track = null;
|
||||
|
||||
/**
|
||||
* Stores some canvas/frame data per resolution/mirror states
|
||||
*/
|
||||
this._workspace = [];
|
||||
}
|
||||
|
||||
static get FORMAT_IMAGE_DATA () {
|
||||
return 'image-data';
|
||||
}
|
||||
|
||||
static get FORMAT_CANVAS () {
|
||||
return 'canvas';
|
||||
}
|
||||
|
||||
/**
|
||||
* Dimensions the video stream is analyzed at after its rendered to the
|
||||
* sample canvas.
|
||||
* @type {Array.<number>}
|
||||
*/
|
||||
static get DIMENSIONS () {
|
||||
return [480, 360];
|
||||
}
|
||||
|
||||
/**
|
||||
* Order preview drawable is inserted at in the renderer.
|
||||
* @type {number}
|
||||
*/
|
||||
static get ORDER () {
|
||||
return 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the HTML video element containing the stream
|
||||
*/
|
||||
get video () {
|
||||
return this._video;
|
||||
}
|
||||
|
||||
/**
|
||||
* Request video be enabled. Sets up video, creates video skin and enables preview.
|
||||
*
|
||||
* @return {Promise.<Video>} resolves a promise to this video provider when video is ready.
|
||||
*/
|
||||
enableVideo () {
|
||||
this.enabled = true;
|
||||
return this._setupVideo();
|
||||
}
|
||||
|
||||
/**
|
||||
* Disable video stream (turn video off)
|
||||
*/
|
||||
disableVideo () {
|
||||
this.enabled = false;
|
||||
// If we have begun a setup process, call _teardown after it completes
|
||||
if (this._singleSetup) {
|
||||
this._singleSetup
|
||||
.then(this._teardown.bind(this))
|
||||
.catch(err => this.onError(err));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* async part of disableVideo
|
||||
* @private
|
||||
*/
|
||||
_teardown () {
|
||||
// we might be asked to re-enable before _teardown is called, just ignore it.
|
||||
if (this.enabled === false) {
|
||||
const disableTrack = requestDisableVideo();
|
||||
this._singleSetup = null;
|
||||
// by clearing refs to video and track, we should lose our hold over the camera
|
||||
this._video = null;
|
||||
if (this._track && disableTrack) {
|
||||
this._track.stop();
|
||||
}
|
||||
this._track = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Return frame data from the video feed in a specified dimensions, format, and mirroring.
|
||||
*
|
||||
* @param {object} frameInfo A descriptor of the frame you would like to receive.
|
||||
* @param {Array.<number>} frameInfo.dimensions [width, height] array of numbers. Defaults to [480,360]
|
||||
* @param {boolean} frameInfo.mirror If you specificly want a mirror/non-mirror frame, defaults to true
|
||||
* @param {string} frameInfo.format Requested video format, available formats are 'image-data' and 'canvas'.
|
||||
* @param {number} frameInfo.cacheTimeout Will reuse previous image data if the time since capture is less than
|
||||
* the cacheTimeout. Defaults to 16ms.
|
||||
*
|
||||
* @return {ArrayBuffer|Canvas|string|null} Frame data in requested format, null when errors.
|
||||
*/
|
||||
getFrame ({
|
||||
dimensions = VideoProvider.DIMENSIONS,
|
||||
mirror = this.mirror,
|
||||
format = VideoProvider.FORMAT_IMAGE_DATA,
|
||||
cacheTimeout = this._frameCacheTimeout
|
||||
}) {
|
||||
if (!this.videoReady) {
|
||||
return null;
|
||||
}
|
||||
const [width, height] = dimensions;
|
||||
const workspace = this._getWorkspace({dimensions, mirror: Boolean(mirror)});
|
||||
const {videoWidth, videoHeight} = this._video;
|
||||
const {canvas, context, lastUpdate, cacheData} = workspace;
|
||||
const now = Date.now();
|
||||
|
||||
// if the canvas hasn't been updated...
|
||||
if (lastUpdate + cacheTimeout < now) {
|
||||
|
||||
if (mirror) {
|
||||
context.scale(-1, 1);
|
||||
context.translate(width * -1, 0);
|
||||
}
|
||||
|
||||
context.drawImage(this._video,
|
||||
// source x, y, width, height
|
||||
0, 0, videoWidth, videoHeight,
|
||||
// dest x, y, width, height
|
||||
0, 0, width, height
|
||||
);
|
||||
|
||||
// context.resetTransform() doesn't work on Edge but the following should
|
||||
context.setTransform(1, 0, 0, 1, 0, 0);
|
||||
workspace.lastUpdate = now;
|
||||
}
|
||||
|
||||
// each data type has it's own data cache, but the canvas is the same
|
||||
if (!cacheData[format]) {
|
||||
cacheData[format] = {lastUpdate: 0};
|
||||
}
|
||||
const formatCache = cacheData[format];
|
||||
|
||||
if (formatCache.lastUpdate + cacheTimeout < now) {
|
||||
if (format === VideoProvider.FORMAT_IMAGE_DATA) {
|
||||
formatCache.lastData = context.getImageData(0, 0, width, height);
|
||||
} else if (format === VideoProvider.FORMAT_CANVAS) {
|
||||
// this will never change
|
||||
formatCache.lastUpdate = Infinity;
|
||||
formatCache.lastData = canvas;
|
||||
} else {
|
||||
log.error(`video io error - unimplemented format ${format}`);
|
||||
// cache the null result forever, don't log about it again..
|
||||
formatCache.lastUpdate = Infinity;
|
||||
formatCache.lastData = null;
|
||||
}
|
||||
|
||||
// rather than set to now, this data is as stale as it's canvas is
|
||||
formatCache.lastUpdate = Math.max(workspace.lastUpdate, formatCache.lastUpdate);
|
||||
}
|
||||
|
||||
return formatCache.lastData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Method called when an error happens. Default implementation is just to log error.
|
||||
*
|
||||
* @abstract
|
||||
* @param {Error} error An error object from getUserMedia or other source of error.
|
||||
*/
|
||||
onError (error) {
|
||||
log.error('Unhandled video io device error', error);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a video stream.
|
||||
* @private
|
||||
* @return {Promise} When video has been received, rejected if video is not received
|
||||
*/
|
||||
_setupVideo () {
|
||||
// We cache the result of this setup so that we can only ever have a single
|
||||
// video/getUserMedia request happen at a time.
|
||||
if (this._singleSetup) {
|
||||
return this._singleSetup;
|
||||
}
|
||||
|
||||
this._singleSetup = requestVideoStream({
|
||||
width: {min: 480, ideal: 640},
|
||||
height: {min: 360, ideal: 480}
|
||||
})
|
||||
.then(stream => {
|
||||
this._video = document.createElement('video');
|
||||
|
||||
// Use the new srcObject API, falling back to createObjectURL
|
||||
try {
|
||||
this._video.srcObject = stream;
|
||||
} catch (error) {
|
||||
this._video.src = window.URL.createObjectURL(stream);
|
||||
}
|
||||
// Hint to the stream that it should load. A standard way to do this
|
||||
// is add the video tag to the DOM. Since this extension wants to
|
||||
// hide the video tag and instead render a sample of the stream into
|
||||
// the webgl rendered Scratch canvas, another hint like this one is
|
||||
// needed.
|
||||
this._video.play(); // Needed for Safari/Firefox, Chrome auto-plays.
|
||||
this._track = stream.getTracks()[0];
|
||||
return this;
|
||||
})
|
||||
.catch(error => {
|
||||
this._singleSetup = null;
|
||||
this.onError(error);
|
||||
});
|
||||
|
||||
return this._singleSetup;
|
||||
}
|
||||
|
||||
get videoReady () {
|
||||
if (!this.enabled) {
|
||||
return false;
|
||||
}
|
||||
if (!this._video) {
|
||||
return false;
|
||||
}
|
||||
if (!this._track) {
|
||||
return false;
|
||||
}
|
||||
const {videoWidth, videoHeight} = this._video;
|
||||
if (typeof videoWidth !== 'number' || typeof videoHeight !== 'number') {
|
||||
return false;
|
||||
}
|
||||
if (videoWidth === 0 || videoHeight === 0) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* get an internal workspace for canvas/context/caches
|
||||
* this uses some document stuff to create a canvas and what not, probably needs abstraction
|
||||
* into the renderer layer?
|
||||
* @private
|
||||
* @return {object} A workspace for canvas/data storage. Internal format not documented intentionally
|
||||
*/
|
||||
_getWorkspace ({dimensions, mirror}) {
|
||||
let workspace = this._workspace.find(space => (
|
||||
space.dimensions.join('-') === dimensions.join('-') &&
|
||||
space.mirror === mirror
|
||||
));
|
||||
if (!workspace) {
|
||||
workspace = {
|
||||
dimensions,
|
||||
mirror,
|
||||
canvas: document.createElement('canvas'),
|
||||
lastUpdate: 0,
|
||||
cacheData: {}
|
||||
};
|
||||
workspace.canvas.width = dimensions[0];
|
||||
workspace.canvas.height = dimensions[1];
|
||||
workspace.context = workspace.canvas.getContext('2d');
|
||||
this._workspace.push(workspace);
|
||||
}
|
||||
return workspace;
|
||||
}
|
||||
}
|
||||
|
||||
export default VideoProvider;
|
||||
Reference in new issue
Block a user