diff --git a/package.json b/package.json index 8841cf721..28df3b4d5 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "babel-loader": "^8.0.4", "base64-loader": "1.0.0", "bowser": "1.9.4", - "chromedriver": "74.0.0", + "chromedriver": "75.1.0", "classnames": "2.2.6", "computed-style-to-inline-style": "3.0.0", "copy-webpack-plugin": "^4.5.1", diff --git a/src/components/asset-panel/asset-panel.css b/src/components/asset-panel/asset-panel.css index bfd578909..b974fea6b 100644 --- a/src/components/asset-panel/asset-panel.css +++ b/src/components/asset-panel/asset-panel.css @@ -24,7 +24,7 @@ display: flex; flex-grow: 1; flex-shrink: 1; - overflow-y: auto; + overflow: visible; } [dir="ltr"] .detail-area { diff --git a/src/components/audio-trimmer/audio-selector.jsx b/src/components/audio-trimmer/audio-selector.jsx new file mode 100644 index 000000000..a7c0b41c2 --- /dev/null +++ b/src/components/audio-trimmer/audio-selector.jsx @@ -0,0 +1,53 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import classNames from 'classnames'; +import Box from '../box/box.jsx'; +import styles from './audio-trimmer.css'; +import SelectionHandle from './selection-handle.jsx'; +import Playhead from './playhead.jsx'; + +const AudioSelector = props => ( +
+ {props.trimStart === null ? null : ( + + + + + + )} + {props.playhead ? ( + + ) : null} +
+); + +AudioSelector.propTypes = { + containerRef: PropTypes.func, + onNewSelectionMouseDown: PropTypes.func.isRequired, + onTrimEndMouseDown: PropTypes.func.isRequired, + onTrimStartMouseDown: PropTypes.func.isRequired, + playhead: PropTypes.number, + trimEnd: PropTypes.number, + trimStart: PropTypes.number +}; + +export default AudioSelector; diff --git a/src/components/audio-trimmer/audio-trimmer.css b/src/components/audio-trimmer/audio-trimmer.css index 63af2a0cb..f3ce9fa42 100644 --- a/src/components/audio-trimmer/audio-trimmer.css +++ b/src/components/audio-trimmer/audio-trimmer.css @@ -1,10 +1,11 @@ @import "../../css/colors.css"; $border-radius: 4px; -$trim-handle-width: 12px; -$trim-handle-height: 14px; +$trim-handle-width: 30px; +$trim-handle-height: 30px; +$trim-handle-border: 3px; $stripe-size: 10px; -$hover-scale: 2; +$hover-scale: 1.25; .absolute { position: absolute; @@ -17,6 +18,10 @@ $hover-scale: 2; transform: translateZ(0); } +.selector { + cursor: pointer; +} + .trim-background { cursor: pointer; touch-action: none; @@ -35,6 +40,11 @@ $hover-scale: 2; ); } +.selection-background { + background: $motion-primary; + opacity: 0.5; +} + .start-trim-background .trim-background-mask { border-top-left-radius: $border-radius; border-bottom-left-radius: $border-radius; @@ -53,6 +63,10 @@ $hover-scale: 2; border: 1px solid $red-tertiary; } +.selector .trim-line { + border: 1px solid $motion-tertiary; +} + .playhead-container { position: absolute; top: 0; @@ -68,31 +82,50 @@ $hover-scale: 2; so that we can use transform: translateX() using percentages. */ width: 100%; + height: 100%; border-left: 1px solid $motion-primary; border-top: none; border-bottom: none; border-right: none; } -.start-trim-line { - right: 0; +.right-handle { + transform: scaleX(-1); } -.end-trim-line { - left: 0; +.selector .left-handle { + left: -1px +} + +.selector .right-handle { + right: -1px +} + +.trimmer .left-handle { + right: -1px +} + +.trimmer .right-handle { + left: -1px } .trim-handle { position: absolute; - left: calc(-$trim-handle-width / 2); width: $trim-handle-width; height: $trim-handle-height; + right: 0; + user-select: none; +} + +.trimmer .trim-handle { + filter: hue-rotate(150deg); } .trim-handle img { position: absolute; width: $trim-handle-width; height: $trim-handle-height; + left: calc($trim-handle-border * 1.5); /* Make sure image dragging isn't triggered */ user-select: none; @@ -103,22 +136,13 @@ $hover-scale: 2; } .top-trim-handle { - top: -$trim-handle-height; + top: calc(-$trim-handle-height + $trim-handle-border); } .bottom-trim-handle { - bottom: -$trim-handle-height; + bottom: calc(-$trim-handle-height + $trim-handle-border); } .top-trim-handle img { - transform: rotate(180deg); -} - -/* Increase handle size when anywhere on draggable area is hovered */ -.trim-background:hover img { - transform: scale($hover-scale); -} - -.trim-background:hover .top-trim-handle img { - transform: rotate(180deg) scale($hover-scale); + transform: scaleY(-1); } diff --git a/src/components/audio-trimmer/audio-trimmer.jsx b/src/components/audio-trimmer/audio-trimmer.jsx index 87ca75a98..d9874ddd1 100644 --- a/src/components/audio-trimmer/audio-trimmer.jsx +++ b/src/components/audio-trimmer/audio-trimmer.jsx @@ -3,11 +3,12 @@ import React from 'react'; import classNames from 'classnames'; import Box from '../box/box.jsx'; import styles from './audio-trimmer.css'; -import handleIcon from './icon--handle.svg'; +import SelectionHandle from './selection-handle.jsx'; +import Playhead from './playhead.jsx'; const AudioTrimmer = props => (
{props.trimStart === null ? null : ( @@ -20,28 +21,16 @@ const AudioTrimmer = props => ( onTouchStart={props.onTrimStartMouseDown} > - - - - - - - - + )} - {props.playhead ? ( -
-
-
+ ) : null} - {props.trimEnd === null ? null : ( ( onTouchStart={props.onTrimEndMouseDown} > - - - - - - - - + )}
diff --git a/src/components/audio-trimmer/icon--handle.svg b/src/components/audio-trimmer/icon--handle.svg index a3f52078f..dad1ea751 100644 --- a/src/components/audio-trimmer/icon--handle.svg +++ b/src/components/audio-trimmer/icon--handle.svg @@ -1,18 +1,16 @@ - - - - handle + + + + Bottom Left Handle Created with Sketch. - - - - - - - - - - + + + + + + + + diff --git a/src/components/audio-trimmer/playhead.jsx b/src/components/audio-trimmer/playhead.jsx new file mode 100644 index 000000000..be5423809 --- /dev/null +++ b/src/components/audio-trimmer/playhead.jsx @@ -0,0 +1,21 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import classNames from 'classnames'; +import styles from './audio-trimmer.css'; + +const Playhead = props => ( +
+
+
+); + +Playhead.propTypes = { + playbackPosition: PropTypes.number +}; + +export default Playhead; diff --git a/src/components/audio-trimmer/selection-handle.jsx b/src/components/audio-trimmer/selection-handle.jsx new file mode 100644 index 000000000..f182ce6a4 --- /dev/null +++ b/src/components/audio-trimmer/selection-handle.jsx @@ -0,0 +1,28 @@ +import PropTypes from 'prop-types'; +import React from 'react'; +import classNames from 'classnames'; +import Box from '../box/box.jsx'; +import styles from './audio-trimmer.css'; +import handleIcon from './icon--handle.svg'; + +const SelectionHandle = props => ( + + + + + + + + +); + +SelectionHandle.propTypes = { + handleStyle: PropTypes.string, + onMouseDown: PropTypes.func +}; + +export default SelectionHandle; diff --git a/src/components/icon-button/icon-button.css b/src/components/icon-button/icon-button.css index f58ee6f8d..002c817ae 100644 --- a/src/components/icon-button/icon-button.css +++ b/src/components/icon-button/icon-button.css @@ -8,6 +8,7 @@ font-size: 0.75rem; font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; color: $motion-primary; + border-radius: 0.5rem; } .container + .container { @@ -16,4 +17,14 @@ .title { margin-top: 0.5rem; + text-align: center; +} + +.disabled { + opacity: 0.5; + pointer-events: none; +} + +.container:active { + background-color: $motion-light-transparent; } diff --git a/src/components/icon-button/icon-button.jsx b/src/components/icon-button/icon-button.jsx index 832e4a65b..c6549fdf9 100644 --- a/src/components/icon-button/icon-button.jsx +++ b/src/components/icon-button/icon-button.jsx @@ -5,14 +5,19 @@ import styles from './icon-button.css'; const IconButton = ({ img, + disabled, className, title, onClick }) => (
+ + + Sound/General/Copy to New + Created with Sketch. + + + + \ No newline at end of file diff --git a/src/components/sound-editor/icon--copy.svg b/src/components/sound-editor/icon--copy.svg new file mode 100644 index 000000000..7bac62c1b --- /dev/null +++ b/src/components/sound-editor/icon--copy.svg @@ -0,0 +1,14 @@ + + + + Sound/General/Copy + Created with Sketch. + + + + + + + + + \ No newline at end of file diff --git a/src/components/sound-editor/icon--delete.svg b/src/components/sound-editor/icon--delete.svg new file mode 100644 index 000000000..8f662201e --- /dev/null +++ b/src/components/sound-editor/icon--delete.svg @@ -0,0 +1,18 @@ + + + + Sound/General/Trim + Created with Sketch. + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/components/sound-editor/icon--fade-in.svg b/src/components/sound-editor/icon--fade-in.svg new file mode 100644 index 000000000..6c7487584 --- /dev/null +++ b/src/components/sound-editor/icon--fade-in.svg @@ -0,0 +1,14 @@ + + + + Sound/Effects/Fade-in + Created with Sketch. + + + + + + + + + \ No newline at end of file diff --git a/src/components/sound-editor/icon--fade-out.svg b/src/components/sound-editor/icon--fade-out.svg new file mode 100644 index 000000000..96e8d6244 --- /dev/null +++ b/src/components/sound-editor/icon--fade-out.svg @@ -0,0 +1,14 @@ + + + + Sound/Effects/Fade-out + Created with Sketch. + + + + + + + + + \ No newline at end of file diff --git a/src/components/sound-editor/icon--lounder.svg b/src/components/sound-editor/icon--lounder.svg deleted file mode 100644 index a1022b4b8..000000000 --- a/src/components/sound-editor/icon--lounder.svg +++ /dev/null @@ -1,17 +0,0 @@ - - - - icon--lounder - Created with Sketch. - - - - - - - - - - - - \ No newline at end of file diff --git a/src/components/sound-editor/icon--mute.svg b/src/components/sound-editor/icon--mute.svg new file mode 100644 index 000000000..6cdd71d4d --- /dev/null +++ b/src/components/sound-editor/icon--mute.svg @@ -0,0 +1,9 @@ + + + + Sound/Effects/Mute + Created with Sketch. + + + + diff --git a/src/components/sound-editor/icon--paste.svg b/src/components/sound-editor/icon--paste.svg new file mode 100644 index 000000000..53ca104c1 --- /dev/null +++ b/src/components/sound-editor/icon--paste.svg @@ -0,0 +1,9 @@ + + + + Sound/General/Paste + Created with Sketch. + + + + \ No newline at end of file diff --git a/src/components/sound-editor/icon--play.svg b/src/components/sound-editor/icon--play.svg new file mode 100644 index 000000000..817a2cab9 --- /dev/null +++ b/src/components/sound-editor/icon--play.svg @@ -0,0 +1,20 @@ + + + + play + Created with Sketch. + + + + + + + + + + + + + + + diff --git a/src/components/sound-editor/icon--stop.svg b/src/components/sound-editor/icon--stop.svg new file mode 100644 index 000000000..c960e66c3 --- /dev/null +++ b/src/components/sound-editor/icon--stop.svg @@ -0,0 +1,21 @@ + + + + stop-playback + Created with Sketch. + + + + + + + + + + + + + + + + diff --git a/src/components/sound-editor/sound-editor.css b/src/components/sound-editor/sound-editor.css index 279875fc8..7a791e6ab 100644 --- a/src/components/sound-editor/sound-editor.css +++ b/src/components/sound-editor/sound-editor.css @@ -63,7 +63,9 @@ background: hsla(300, 53%, 60%, 0.15); border: 1px solid $ui-black-transparent; border-radius: 5px; - padding: 3px; + + margin-top: 20px; + margin-bottom: 20px; } $border-radius: 0.25rem; @@ -93,9 +95,9 @@ $border-radius: 0.25rem; height: 3rem; width: 3rem; outline: none; - background: white; + background: $motion-primary; border-radius: 100%; - border: 1px solid $ui-black-transparent; + border: 4px solid $ui-white-dim; cursor: pointer; padding: 0.75rem; user-select: none; @@ -118,6 +120,7 @@ $border-radius: 0.25rem; color: $text-primary; font-size: 0.625rem; user-select: none; + } [dir="ltr"] .trim-button { @@ -130,7 +133,6 @@ $border-radius: 0.25rem; .trim-button > img { width: 1.25rem; - margin-bottom: -0.375rem; } .effect-button { @@ -138,6 +140,7 @@ $border-radius: 0.25rem; color: $text-primary; font-size: 0.625rem; user-select: none; + padding: 0.25rem 0; } .effect-button + .effect-button { @@ -150,12 +153,25 @@ $border-radius: 0.25rem; margin-bottom: -0.375rem; } -/* mirror the louder/softer speaker icons when rtl */ -[dir="rtl"] .effect-button:nth-of-type(6) img { - transform: scaleX(-1); +.tool-button { + flex-basis: 60px; + color: $text-primary; + font-size: 0.625rem; + user-select: none; + padding: 0.25rem 0; } -[dir="rtl"] .effect-button:nth-of-type(7) img { +.tool-button + .tool-button { + margin: 0; +} + +.tool-button img { + width: 4rem; + height: 1.5rem; + margin-bottom: -0.375rem; +} + +[dir="rtl"] .flip-in-rtl img { transform: scaleX(-1); } @@ -167,6 +183,10 @@ $border-radius: 0.25rem; margin-right: 1rem; } +.button-group { + display: flex; +} + .button-group .button { border-radius: 0; } diff --git a/src/components/sound-editor/sound-editor.jsx b/src/components/sound-editor/sound-editor.jsx index f97338e06..84f56d08b 100644 --- a/src/components/sound-editor/sound-editor.jsx +++ b/src/components/sound-editor/sound-editor.jsx @@ -8,24 +8,29 @@ import Label from '../forms/label.jsx'; import Input from '../forms/input.jsx'; import BufferedInputHOC from '../forms/buffered-input-hoc.jsx'; -import AudioTrimmer from '../../containers/audio-trimmer.jsx'; +import AudioSelector from '../../containers/audio-selector.jsx'; import IconButton from '../icon-button/icon-button.jsx'; import styles from './sound-editor.css'; -import playIcon from '../record-modal/icon--play.svg'; -import stopIcon from '../record-modal/icon--stop-playback.svg'; -import trimIcon from './icon--trim.svg'; -import trimConfirmIcon from './icon--trim-confirm.svg'; +import playIcon from './icon--play.svg'; +import stopIcon from './icon--stop.svg'; import redoIcon from './icon--redo.svg'; import undoIcon from './icon--undo.svg'; -import echoIcon from './icon--echo.svg'; import fasterIcon from './icon--faster.svg'; import slowerIcon from './icon--slower.svg'; import louderIcon from './icon--louder.svg'; import softerIcon from './icon--softer.svg'; import robotIcon from './icon--robot.svg'; import reverseIcon from './icon--reverse.svg'; +import fadeOutIcon from './icon--fade-out.svg'; +import fadeInIcon from './icon--fade-in.svg'; +import muteIcon from './icon--mute.svg'; + +import deleteIcon from './icon--delete.svg'; +import copyIcon from './icon--copy.svg'; +import pasteIcon from './icon--paste.svg'; +import copyToNewIcon from './icon--copy-to-new.svg'; const BufferedInput = BufferedInputHOC(Input); @@ -45,10 +50,25 @@ const messages = defineMessages({ description: 'Title of the button to stop the sound', defaultMessage: 'Stop' }, - trim: { - id: 'gui.soundEditor.trim', - description: 'Title of the button to start trimminging the sound', - defaultMessage: 'Trim' + copy: { + id: 'gui.soundEditor.copy', + description: 'Title of the button to copy the sound', + defaultMessage: 'Copy' + }, + paste: { + id: 'gui.soundEditor.paste', + description: 'Title of the button to paste the sound', + defaultMessage: 'Paste' + }, + copyToNew: { + id: 'gui.soundEditor.copyToNew', + description: 'Title of the button to copy the selection into a new sound', + defaultMessage: 'Copy to New' + }, + delete: { + id: 'gui.soundEditor.delete', + description: 'Title of the button to delete the sound', + defaultMessage: 'Delete' }, save: { id: 'gui.soundEditor.save', @@ -99,11 +119,30 @@ const messages = defineMessages({ id: 'gui.soundEditor.reverse', description: 'Title of the button to apply the reverse effect', defaultMessage: 'Reverse' + }, + fadeOut: { + id: 'gui.soundEditor.fadeOut', + description: 'Title of the button to apply the fade out effect', + defaultMessage: 'Fade out' + }, + fadeIn: { + id: 'gui.soundEditor.fadeIn', + description: 'Title of the button to apply the fade in effect', + defaultMessage: 'Fade in' + }, + mute: { + id: 'gui.soundEditor.mute', + description: 'Title of the button to apply the mute effect', + defaultMessage: 'Mute' } }); const SoundEditor = props => ( -
+
+
+ + + +
- ) : ( - - )} - onClick={props.onActivateTrim} + className={styles.toolButton} + disabled={props.trimStart === null} + img={deleteIcon} + title={props.intl.formatMessage(messages.delete)} + onClick={props.onDelete} />
@@ -161,12 +216,13 @@ const SoundEditor = props => ( height={160} width={600} /> -
@@ -209,61 +265,81 @@ const SoundEditor = props => ( onClick={props.onSlower} /> } - onClick={props.onEcho} - /> - } - onClick={props.onRobot} - /> - } onClick={props.onLouder} /> } onClick={props.onSofter} /> + } + onClick={props.onMute} + /> + } + onClick={props.onFadeIn} + /> + } + onClick={props.onFadeOut} + /> } onClick={props.onReverse} /> + } + onClick={props.onRobot} + />
); SoundEditor.propTypes = { + canPaste: PropTypes.bool.isRequired, canRedo: PropTypes.bool.isRequired, canUndo: PropTypes.bool.isRequired, chunkLevels: PropTypes.arrayOf(PropTypes.number).isRequired, intl: intlShape, name: PropTypes.string.isRequired, - onActivateTrim: PropTypes.func, onChangeName: PropTypes.func.isRequired, + onContainerClick: PropTypes.func.isRequired, + onCopy: PropTypes.func.isRequired, + onCopyToNew: PropTypes.func.isRequired, + onDelete: PropTypes.func, onEcho: PropTypes.func.isRequired, + onFadeIn: PropTypes.func.isRequired, + onFadeOut: PropTypes.func.isRequired, onFaster: PropTypes.func.isRequired, onLouder: PropTypes.func.isRequired, + onMute: PropTypes.func.isRequired, + onPaste: PropTypes.func.isRequired, onPlay: PropTypes.func.isRequired, onRedo: PropTypes.func.isRequired, onReverse: PropTypes.func.isRequired, onRobot: PropTypes.func.isRequired, - onSetTrimEnd: PropTypes.func, - onSetTrimStart: PropTypes.func, + onSetTrim: PropTypes.func, onSlower: PropTypes.func.isRequired, onSofter: PropTypes.func.isRequired, onStop: PropTypes.func.isRequired, onUndo: PropTypes.func.isRequired, playhead: PropTypes.number, + setRef: PropTypes.func, trimEnd: PropTypes.number, trimStart: PropTypes.number }; diff --git a/src/components/waveform/waveform.jsx b/src/components/waveform/waveform.jsx index 48a0a4935..3c7b4d26b 100644 --- a/src/components/waveform/waveform.jsx +++ b/src/components/waveform/waveform.jsx @@ -19,17 +19,21 @@ class Waveform extends React.PureComponent { // composite time when animating the playhead const takeEveryN = Math.ceil(data.length / width); - const filteredData = takeEveryN === 1 ? data : + const filteredData = takeEveryN === 1 ? data.slice(0) : data.filter((_, i) => i % takeEveryN === 0); - const cappedData = [0, ...filteredData, 0]; + // Need at least two points to render waveform. + if (filteredData.length === 1) { + filteredData.push(filteredData[0]); + } + const maxIndex = filteredData.length - 1; const points = [ - ...cappedData.map((v, i) => - [width * i / cappedData.length, height * v / 2] + ...filteredData.map((v, i) => + [width * (i / maxIndex), height * v / 2] ), - ...cappedData.reverse().map((v, i) => - [width * (cappedData.length - i - 1) / cappedData.length, -height * v / 2] + ...filteredData.reverse().map((v, i) => + [width * (1 - (i / maxIndex)), -height * v / 2] ) ]; const pathComponents = points.map(([x, y], i) => { @@ -40,15 +44,8 @@ class Waveform extends React.PureComponent { return ( - this.initialTrimEnd) { + this.setState({ + trimStart: this.initialTrimEnd, + trimEnd: newTrim + }); + } else { + this.setState({ + trimStart: newTrim, + trimEnd: this.initialTrimEnd + }); + } + } + handleTrimEndMouseMove (currentOffset, initialOffset) { + const dx = (currentOffset.x - initialOffset.x) / this.containerSize; + const newTrim = Math.min(1, Math.max(0, this.initialTrimEnd + dx)); + if (newTrim < this.initialTrimStart) { + this.setState({ + trimStart: newTrim, + trimEnd: this.initialTrimStart + }); + } else { + this.setState({ + trimStart: this.initialTrimStart, + trimEnd: newTrim + }); + } + } + handleTrimStartMouseUp () { + this.props.onSetTrim(this.state.trimStart, this.state.trimEnd); + } + handleTrimEndMouseUp () { + // If the selection was made quickly (tooFast) and is small (tooShort), + // deselect instead. This allows click-to-deselect even if you drag + // a little bit by accident. It also allows very quickly making a + // selection, as long as it is above a minimum length. + const tooFast = (Date.now() - this.clickStartTime) < MIN_DURATION; + const tooShort = (this.state.trimEnd - this.state.trimStart) < MIN_LENGTH; + if (tooFast && tooShort) { + this.clearSelection(); + } else { + this.props.onSetTrim(this.state.trimStart, this.state.trimEnd); + } + } + handleTrimStartMouseDown (e) { + this.containerSize = this.containerElement.getBoundingClientRect().width; + this.trimStartDragRecognizer.start(e); + this.initialTrimStart = this.props.trimStart; + this.initialTrimEnd = this.props.trimEnd; + e.stopPropagation(); + e.preventDefault(); + } + handleTrimEndMouseDown (e) { + this.containerSize = this.containerElement.getBoundingClientRect().width; + this.trimEndDragRecognizer.start(e); + this.initialTrimEnd = this.props.trimEnd; + this.initialTrimStart = this.props.trimStart; + e.stopPropagation(); + e.preventDefault(); + } + storeRef (el) { + this.containerElement = el; + } + render () { + return ( + + ); + } +} + +AudioSelector.propTypes = { + onSetTrim: PropTypes.func, + playhead: PropTypes.number, + trimEnd: PropTypes.number, + trimStart: PropTypes.number +}; + +export default AudioSelector; diff --git a/src/containers/audio-trimmer.jsx b/src/containers/audio-trimmer.jsx index 9cf6e38f1..78bd4ebac 100644 --- a/src/containers/audio-trimmer.jsx +++ b/src/containers/audio-trimmer.jsx @@ -2,7 +2,7 @@ import React from 'react'; import PropTypes from 'prop-types'; import bindAll from 'lodash.bindall'; import AudioTrimmerComponent from '../components/audio-trimmer/audio-trimmer.jsx'; -import {getEventXY} from '../lib/touch-utils'; +import DragRecognizer from '../lib/drag-recognizer'; const MIN_LENGTH = 0.01; // Used to stop sounds being trimmed smaller than 1% @@ -14,52 +14,39 @@ class AudioTrimmer extends React.Component { 'handleTrimEndMouseDown', 'handleTrimStartMouseMove', 'handleTrimEndMouseMove', - 'handleTrimStartMouseUp', - 'handleTrimEndMouseUp', 'storeRef' ]); + this.trimStartDragRecognizer = new DragRecognizer({ + onDrag: this.handleTrimStartMouseMove, + touchDragAngle: 90, + distanceThreshold: 0 + }); + this.trimEndDragRecognizer = new DragRecognizer({ + onDrag: this.handleTrimEndMouseMove, + touchDragAngle: 90, + distanceThreshold: 0 + }); + } - handleTrimStartMouseMove (e) { - const containerSize = this.containerElement.getBoundingClientRect().width; - const dx = (getEventXY(e).x - this.initialX) / containerSize; + handleTrimStartMouseMove (currentOffset, initialOffset) { + const dx = (currentOffset.x - initialOffset.x) / this.containerSize; const newTrim = Math.max(0, Math.min(this.props.trimEnd - MIN_LENGTH, this.initialTrim + dx)); this.props.onSetTrimStart(newTrim); - e.preventDefault(); } - handleTrimEndMouseMove (e) { - const containerSize = this.containerElement.getBoundingClientRect().width; - const dx = (getEventXY(e).x - this.initialX) / containerSize; + handleTrimEndMouseMove (currentOffset, initialOffset) { + const dx = (currentOffset.x - initialOffset.x) / this.containerSize; const newTrim = Math.min(1, Math.max(this.props.trimStart + MIN_LENGTH, this.initialTrim + dx)); this.props.onSetTrimEnd(newTrim); - e.preventDefault(); - } - handleTrimStartMouseUp () { - window.removeEventListener('mousemove', this.handleTrimStartMouseMove); - window.removeEventListener('mouseup', this.handleTrimStartMouseUp); - window.removeEventListener('touchmove', this.handleTrimStartMouseMove); - window.removeEventListener('touchend', this.handleTrimStartMouseUp); - } - handleTrimEndMouseUp () { - window.removeEventListener('mousemove', this.handleTrimEndMouseMove); - window.removeEventListener('mouseup', this.handleTrimEndMouseUp); - window.removeEventListener('touchmove', this.handleTrimEndMouseMove); - window.removeEventListener('touchend', this.handleTrimEndMouseUp); } handleTrimStartMouseDown (e) { - this.initialX = getEventXY(e).x; + this.containerSize = this.containerElement.getBoundingClientRect().width; + this.trimStartDragRecognizer.start(e); this.initialTrim = this.props.trimStart; - window.addEventListener('mousemove', this.handleTrimStartMouseMove); - window.addEventListener('mouseup', this.handleTrimStartMouseUp); - window.addEventListener('touchmove', this.handleTrimStartMouseMove); - window.addEventListener('touchend', this.handleTrimStartMouseUp); } handleTrimEndMouseDown (e) { - this.initialX = getEventXY(e).x; + this.containerSize = this.containerElement.getBoundingClientRect().width; + this.trimEndDragRecognizer.start(e); this.initialTrim = this.props.trimEnd; - window.addEventListener('mousemove', this.handleTrimEndMouseMove); - window.addEventListener('mouseup', this.handleTrimEndMouseUp); - window.addEventListener('touchmove', this.handleTrimEndMouseMove); - window.addEventListener('touchend', this.handleTrimEndMouseUp); } storeRef (el) { this.containerElement = el; diff --git a/src/containers/record-modal.jsx b/src/containers/record-modal.jsx index a7a9e8f83..ffaf968a2 100644 --- a/src/containers/record-modal.jsx +++ b/src/containers/record-modal.jsx @@ -2,8 +2,8 @@ import bindAll from 'lodash.bindall'; import PropTypes from 'prop-types'; import React from 'react'; import VM from 'scratch-vm'; -import WavEncoder from 'wav-encoder'; import {connect} from 'react-redux'; +import {encodeAndAddSoundToVM} from '../lib/audio/audio-util.js'; import RecordModalComponent from '../components/record-modal/record-modal.jsx'; @@ -71,39 +71,12 @@ class RecordModal extends React.Component { const startIndex = Math.floor(this.state.trimStart * sampleCount); const endIndex = Math.floor(this.state.trimEnd * sampleCount); const clippedSamples = this.state.samples.slice(startIndex, endIndex); - WavEncoder.encode({ - sampleRate: this.state.sampleRate, - channelData: [clippedSamples] - }).then(wavBuffer => { - const vmSound = { - format: '', - dataFormat: 'wav', - rate: this.state.sampleRate, - sampleCount: clippedSamples.length - }; - // Create an asset from the encoded .wav and get resulting md5 - const storage = this.props.vm.runtime.storage; - vmSound.asset = storage.createAsset( - storage.AssetType.Sound, - storage.DataFormat.WAV, - new Uint8Array(wavBuffer), - null, - true // generate md5 - ); - vmSound.assetId = vmSound.asset.assetId; - - // update vmSound object with md5 property - vmSound.md5 = `${vmSound.assetId}.${vmSound.dataFormat}`; - // The VM will update the sound name to a fresh name - // if the following is already taken - vmSound.name = 'recording1'; - - this.props.vm.addSound(vmSound).then(() => { + encodeAndAddSoundToVM(this.props.vm, clippedSamples, this.state.sampleRate, 'recording1', + () => { this.props.onClose(); this.props.onNewSound(); }); - }); }); } handleCancel () { diff --git a/src/containers/sound-editor.jsx b/src/containers/sound-editor.jsx index a3cdaaf67..a2b19719f 100644 --- a/src/containers/sound-editor.jsx +++ b/src/containers/sound-editor.jsx @@ -2,10 +2,11 @@ import bindAll from 'lodash.bindall'; import PropTypes from 'prop-types'; import React from 'react'; import WavEncoder from 'wav-encoder'; +import VM from 'scratch-vm'; import {connect} from 'react-redux'; -import {computeChunkedRMS, SOUND_BYTE_LIMIT} from '../lib/audio/audio-util.js'; +import {computeChunkedRMS, encodeAndAddSoundToVM, SOUND_BYTE_LIMIT} from '../lib/audio/audio-util.js'; import AudioEffects from '../lib/audio/audio-effects.js'; import SoundEditorComponent from '../components/sound-editor/sound-editor.jsx'; import AudioBufferPlayer from '../lib/audio/audio-buffer-player.js'; @@ -17,21 +18,29 @@ class SoundEditor extends React.Component { constructor (props) { super(props); bindAll(this, [ + 'copy', 'copyCurrentBuffer', + 'handleCopyToNew', 'handleStoppedPlaying', 'handleChangeName', 'handlePlay', 'handleStopPlaying', 'handleUpdatePlayhead', - 'handleActivateTrim', - 'handleUpdateTrimEnd', - 'handleUpdateTrimStart', + 'handleDelete', + 'handleUpdateTrim', 'handleEffect', 'handleUndo', 'handleRedo', - 'submitNewSamples' + 'submitNewSamples', + 'handleCopy', + 'handlePaste', + 'paste', + 'handleKeyPress', + 'handleContainerClick', + 'setRef' ]); this.state = { + copyBuffer: null, chunkLevels: computeChunkedRMS(this.props.samples), playhead: null, // null is not playing, [0 -> 1] is playing percent trimStart: null, @@ -40,28 +49,84 @@ class SoundEditor extends React.Component { this.redoStack = []; this.undoStack = []; + + this.ref = null; } componentDidMount () { this.audioBufferPlayer = new AudioBufferPlayer(this.props.samples, this.props.sampleRate); + + document.addEventListener('keydown', this.handleKeyPress); } componentWillReceiveProps (newProps) { if (newProps.soundId !== this.props.soundId) { // A different sound has been selected this.redoStack = []; this.undoStack = []; this.resetState(newProps.samples, newProps.sampleRate); + this.setState({ + trimStart: null, + trimEnd: null + }); } } componentWillUnmount () { this.audioBufferPlayer.stop(); + + document.removeEventListener('keydown', this.handleKeyPress); + } + handleKeyPress (event) { + if (event.target instanceof HTMLInputElement) { + // Ignore keyboard shortcuts if a text input field is focused + return; + } + if (event.key === ' ') { + event.preventDefault(); + if (this.state.playhead) { + this.handleStopPlaying(); + } else { + this.handlePlay(); + } + } + if (event.key === 'Delete' || event.key === 'Backspace') { + event.preventDefault(); + if (event.shiftKey) { + this.handleDeleteInverse(); + } else { + this.handleDelete(); + } + } + if (event.key === 'Escape') { + event.preventDefault(); + this.handleUpdateTrim(null, null); + } + if (event.metaKey || event.ctrlKey) { + if (event.shiftKey && event.key.toLowerCase() === 'z') { + event.preventDefault(); + if (this.redoStack.length > 0) { + this.handleRedo(); + } + } else if (event.key === 'z') { + if (this.undoStack.length > 0) { + event.preventDefault(); + this.handleUndo(); + } + } else if (event.key === 'c') { + event.preventDefault(); + this.handleCopy(); + } else if (event.key === 'v') { + event.preventDefault(); + this.handlePaste(); + } else if (event.key === 'a') { + event.preventDefault(); + this.handleUpdateTrim(0, 1); + } + } } resetState (samples, sampleRate) { this.audioBufferPlayer.stop(); this.audioBufferPlayer = new AudioBufferPlayer(samples, sampleRate); this.setState({ chunkLevels: computeChunkedRMS(samples), - playhead: null, - trimStart: null, - trimEnd: null + playhead: null }); } submitNewSamples (samples, sampleRate, skipUndo) { @@ -93,7 +158,7 @@ class SoundEditor extends React.Component { if (this.undoStack.length >= UNDO_STACK_SIZE) { this.undoStack.shift(); // Drop the first element off the array } - this.undoStack.push(this.copyCurrentBuffer()); + this.undoStack.push(this.getUndoItem()); } this.resetState(samples, sampleRate); this.props.vm.updateSoundBuffer( @@ -106,6 +171,7 @@ class SoundEditor extends React.Component { return false; // Update failed } handlePlay () { + this.audioBufferPlayer.stop(); this.audioBufferPlayer.play( this.state.trimStart || 0, this.state.trimEnd || 1, @@ -125,31 +191,46 @@ class SoundEditor extends React.Component { handleChangeName (name) { this.props.vm.renameSound(this.props.soundIndex, name); } - handleActivateTrim () { - if (this.state.trimStart === null && this.state.trimEnd === null) { - this.setState({trimEnd: 0.95, trimStart: 0.05}); + handleDelete () { + const {samples, sampleRate} = this.copyCurrentBuffer(); + const sampleCount = samples.length; + const startIndex = Math.floor(this.state.trimStart * sampleCount); + const endIndex = Math.floor(this.state.trimEnd * sampleCount); + const firstPart = samples.slice(0, startIndex); + const secondPart = samples.slice(endIndex, sampleCount); + const newLength = firstPart.length + secondPart.length; + let newSamples; + if (newLength === 0) { + newSamples = new Float32Array(1); } else { - const {samples, sampleRate} = this.copyCurrentBuffer(); - const sampleCount = samples.length; - const startIndex = Math.floor(this.state.trimStart * sampleCount); - const endIndex = Math.floor(this.state.trimEnd * sampleCount); - if (endIndex > startIndex) { // Strictly greater to prevent 0 sample sounds - const clippedSamples = samples.slice(startIndex, endIndex); - this.submitNewSamples(clippedSamples, sampleRate); - } else { - // Just clear the trim state, it cannot be completed - this.setState({ - trimStart: null, - trimEnd: null - }); - } + newSamples = new Float32Array(newLength); + newSamples.set(firstPart, 0); + newSamples.set(secondPart, firstPart.length); } + this.submitNewSamples(newSamples, sampleRate); + this.setState({ + trimStart: null, + trimEnd: null + }); } - handleUpdateTrimEnd (trimEnd) { - this.setState({trimEnd}); + handleDeleteInverse () { + const {samples, sampleRate} = this.copyCurrentBuffer(); + const sampleCount = samples.length; + const startIndex = Math.floor(this.state.trimStart * sampleCount); + const endIndex = Math.floor(this.state.trimEnd * sampleCount); + let clippedSamples = samples.slice(startIndex, endIndex); + if (clippedSamples.length === 0) { + clippedSamples = new Float32Array(1); + } + this.submitNewSamples(clippedSamples, sampleRate); + this.setState({ + trimStart: null, + trimEnd: null + }); } - handleUpdateTrimStart (trimStart) { - this.setState({trimStart}); + handleUpdateTrim (trimStart, trimEnd) { + this.setState({trimStart, trimEnd}); + this.handleStopPlaying(); } effectFactory (name) { return () => this.handleEffect(name); @@ -162,52 +243,181 @@ class SoundEditor extends React.Component { }; } handleEffect (name) { - const effects = new AudioEffects(this.audioBufferPlayer.buffer, name); - effects.process(({renderedBuffer}) => { + const trimStart = this.state.trimStart === null ? 0.0 : this.state.trimStart; + const trimEnd = this.state.trimEnd === null ? 1.0 : this.state.trimEnd; + + // Offline audio context needs at least 2 samples + if (this.audioBufferPlayer.buffer.length < 2) { + return; + } + + const effects = new AudioEffects(this.audioBufferPlayer.buffer, name, trimStart, trimEnd); + effects.process((renderedBuffer, adjustedTrimStart, adjustedTrimEnd) => { const samples = renderedBuffer.getChannelData(0); const sampleRate = renderedBuffer.sampleRate; const success = this.submitNewSamples(samples, sampleRate); - if (success) this.handlePlay(); + if (success) { + if (this.state.trimStart === null) { + this.handlePlay(); + } else { + this.setState({trimStart: adjustedTrimStart, trimEnd: adjustedTrimEnd}, this.handlePlay); + } + } }); } + getUndoItem () { + return { + ...this.copyCurrentBuffer(), + trimStart: this.state.trimStart, + trimEnd: this.state.trimEnd + }; + } handleUndo () { - this.redoStack.push(this.copyCurrentBuffer()); - const {samples, sampleRate} = this.undoStack.pop(); + this.redoStack.push(this.getUndoItem()); + const {samples, sampleRate, trimStart, trimEnd} = this.undoStack.pop(); if (samples) { this.submitNewSamples(samples, sampleRate, true); - this.handlePlay(); + this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay); } } handleRedo () { - const {samples, sampleRate} = this.redoStack.pop(); + const {samples, sampleRate, trimStart, trimEnd} = this.redoStack.pop(); if (samples) { - this.undoStack.push(this.copyCurrentBuffer()); + this.undoStack.push(this.getUndoItem()); this.submitNewSamples(samples, sampleRate, true); + this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay); + } + } + handleCopy () { + this.copy(); + } + copy (callback) { + const trimStart = this.state.trimStart === null ? 0.0 : this.state.trimStart; + const trimEnd = this.state.trimEnd === null ? 1.0 : this.state.trimEnd; + + const newCopyBuffer = this.copyCurrentBuffer(); + const trimStartSamples = trimStart * newCopyBuffer.samples.length; + const trimEndSamples = trimEnd * newCopyBuffer.samples.length; + newCopyBuffer.samples = newCopyBuffer.samples.slice(trimStartSamples, trimEndSamples); + + this.setState({ + copyBuffer: newCopyBuffer + }, callback); + } + handleCopyToNew () { + this.copy(() => { + encodeAndAddSoundToVM(this.props.vm, this.state.copyBuffer.samples, + this.state.copyBuffer.sampleRate, this.props.name); + }); + } + resampleBufferToRate (buffer, newRate) { + return new Promise(resolve => { + if (window.OfflineAudioContext) { + const sampleRateRatio = newRate / buffer.sampleRate; + const newLength = sampleRateRatio * buffer.samples.length; + const offlineContext = new window.OfflineAudioContext(1, newLength, newRate); + const source = offlineContext.createBufferSource(); + const audioBuffer = offlineContext.createBuffer(1, buffer.samples.length, buffer.sampleRate); + audioBuffer.getChannelData(0).set(buffer.samples); + source.buffer = audioBuffer; + source.connect(offlineContext.destination); + source.start(); + offlineContext.startRendering(); + offlineContext.oncomplete = ({renderedBuffer}) => { + resolve({ + samples: renderedBuffer.getChannelData(0), + sampleRate: newRate + }); + }; + } + }); + } + paste () { + // If there's no selection, paste at the end of the sound + const {samples} = this.copyCurrentBuffer(); + if (this.state.trimStart === null) { + const newLength = samples.length + this.state.copyBuffer.samples.length; + const newSamples = new Float32Array(newLength); + newSamples.set(samples, 0); + newSamples.set(this.state.copyBuffer.samples, samples.length); + this.submitNewSamples(newSamples, this.props.sampleRate, false); this.handlePlay(); + } else { + // else replace the selection with the pasted sound + const trimStartSamples = this.state.trimStart * samples.length; + const trimEndSamples = this.state.trimEnd * samples.length; + const firstPart = samples.slice(0, trimStartSamples); + const lastPart = samples.slice(trimEndSamples); + const newLength = firstPart.length + this.state.copyBuffer.samples.length + lastPart.length; + const newSamples = new Float32Array(newLength); + newSamples.set(firstPart, 0); + newSamples.set(this.state.copyBuffer.samples, firstPart.length); + newSamples.set(lastPart, firstPart.length + this.state.copyBuffer.samples.length); + + const trimStartSeconds = trimStartSamples / this.props.sampleRate; + const trimEndSeconds = trimStartSeconds + + (this.state.copyBuffer.samples.length / this.state.copyBuffer.sampleRate); + const newDurationSeconds = newSamples.length / this.state.copyBuffer.sampleRate; + const adjustedTrimStart = trimStartSeconds / newDurationSeconds; + const adjustedTrimEnd = trimEndSeconds / newDurationSeconds; + this.submitNewSamples(newSamples, this.props.sampleRate, false); + this.setState({ + trimStart: adjustedTrimStart, + trimEnd: adjustedTrimEnd + }, this.handlePlay); + } + } + handlePaste () { + if (!this.state.copyBuffer) return; + if (this.state.copyBuffer.sampleRate === this.props.sampleRate) { + this.paste(); + } else { + this.resampleBufferToRate(this.state.copyBuffer, this.props.sampleRate).then(buffer => { + this.setState({ + copyBuffer: buffer + }, this.paste); + }); + } + } + setRef (element) { + this.ref = element; + } + handleContainerClick (e) { + // If the click is on the sound editor's div (and not any other element), delesect + if (e.target === this.ref && this.state.trimStart !== null) { + this.handleUpdateTrim(null, null); } } render () { const {effectTypes} = AudioEffects; return ( 0} canUndo={this.undoStack.length > 0} chunkLevels={this.state.chunkLevels} name={this.props.name} playhead={this.state.playhead} + setRef={this.setRef} trimEnd={this.state.trimEnd} trimStart={this.state.trimStart} - onActivateTrim={this.handleActivateTrim} onChangeName={this.handleChangeName} + onContainerClick={this.handleContainerClick} + onCopy={this.handleCopy} + onCopyToNew={this.handleCopyToNew} + onDelete={this.handleDelete} onEcho={this.effectFactory(effectTypes.ECHO)} + onFadeIn={this.effectFactory(effectTypes.FADEIN)} + onFadeOut={this.effectFactory(effectTypes.FADEOUT)} onFaster={this.effectFactory(effectTypes.FASTER)} onLouder={this.effectFactory(effectTypes.LOUDER)} + onMute={this.effectFactory(effectTypes.MUTE)} + onPaste={this.handlePaste} onPlay={this.handlePlay} onRedo={this.handleRedo} onReverse={this.effectFactory(effectTypes.REVERSE)} onRobot={this.effectFactory(effectTypes.ROBOT)} - onSetTrimEnd={this.handleUpdateTrimEnd} - onSetTrimStart={this.handleUpdateTrimStart} + onSetTrim={this.handleUpdateTrim} onSlower={this.effectFactory(effectTypes.SLOWER)} onSofter={this.effectFactory(effectTypes.SOFTER)} onStop={this.handleStopPlaying} @@ -223,10 +433,7 @@ SoundEditor.propTypes = { samples: PropTypes.instanceOf(Float32Array), soundId: PropTypes.string, soundIndex: PropTypes.number, - vm: PropTypes.shape({ - updateSoundBuffer: PropTypes.func, - renameSound: PropTypes.func - }) + vm: PropTypes.instanceOf(VM).isRequired }; const mapStateToProps = (state, {soundIndex}) => { diff --git a/src/lib/audio/audio-effects.js b/src/lib/audio/audio-effects.js index ab170f9c6..8dd551f38 100644 --- a/src/lib/audio/audio-effects.js +++ b/src/lib/audio/audio-effects.js @@ -1,6 +1,8 @@ import EchoEffect from './effects/echo-effect.js'; import RobotEffect from './effects/robot-effect.js'; import VolumeEffect from './effects/volume-effect.js'; +import FadeEffect from './effects/fade-effect.js'; +import MuteEffect from './effects/mute-effect.js'; const effectTypes = { ROBOT: 'robot', @@ -9,32 +11,56 @@ const effectTypes = { SOFTER: 'lower', FASTER: 'faster', SLOWER: 'slower', - ECHO: 'echo' + ECHO: 'echo', + FADEIN: 'fade in', + FADEOUT: 'fade out', + MUTE: 'mute' }; class AudioEffects { static get effectTypes () { return effectTypes; } - constructor (buffer, name) { + constructor (buffer, name, trimStart, trimEnd) { + this.trimStartSeconds = (trimStart * buffer.length) / buffer.sampleRate; + this.trimEndSeconds = (trimEnd * buffer.length) / buffer.sampleRate; + this.adjustedTrimStartSeconds = this.trimStartSeconds; + this.adjustedTrimEndSeconds = this.trimEndSeconds; + // Some effects will modify the playback rate and/or number of samples. // Need to precompute those values to create the offline audio context. const pitchRatio = Math.pow(2, 4 / 12); // A major third let sampleCount = buffer.length; - let playbackRate = 1; + const affectedSampleCount = Math.floor((this.trimEndSeconds - this.trimStartSeconds) * + buffer.sampleRate); + let adjustedAffectedSampleCount = affectedSampleCount; + const unaffectedSampleCount = sampleCount - affectedSampleCount; + + this.playbackRate = 1; switch (name) { case effectTypes.ECHO: - sampleCount = buffer.length + (0.25 * 3 * buffer.sampleRate); + sampleCount = Math.max(sampleCount, + Math.floor((this.trimEndSeconds + EchoEffect.TAIL_SECONDS) * buffer.sampleRate)); break; case effectTypes.FASTER: - playbackRate = pitchRatio; - sampleCount = Math.floor(buffer.length / playbackRate); + this.playbackRate = pitchRatio; + adjustedAffectedSampleCount = Math.floor(affectedSampleCount / this.playbackRate); + sampleCount = unaffectedSampleCount + adjustedAffectedSampleCount; + break; case effectTypes.SLOWER: - playbackRate = 1 / pitchRatio; - sampleCount = Math.floor(buffer.length / playbackRate); + this.playbackRate = 1 / pitchRatio; + adjustedAffectedSampleCount = Math.floor(affectedSampleCount / this.playbackRate); + sampleCount = unaffectedSampleCount + adjustedAffectedSampleCount; break; } + + const durationSeconds = sampleCount / buffer.sampleRate; + this.adjustedTrimEndSeconds = this.trimStartSeconds + + (adjustedAffectedSampleCount / buffer.sampleRate); + this.adjustedTrimStart = this.adjustedTrimStartSeconds / durationSeconds; + this.adjustedTrimEnd = this.adjustedTrimEndSeconds / durationSeconds; + if (window.OfflineAudioContext) { this.audioContext = new window.OfflineAudioContext(1, sampleCount, buffer.sampleRate); } else { @@ -52,8 +78,17 @@ class AudioEffects { const newBuffer = this.audioContext.createBuffer(1, buffer.length, buffer.sampleRate); const newBufferData = newBuffer.getChannelData(0); const bufferLength = buffer.length; + + const startSamples = Math.floor(this.trimStartSeconds * buffer.sampleRate); + const endSamples = Math.floor(this.trimEndSeconds * buffer.sampleRate); + let counter = 0; for (let i = 0; i < bufferLength; i++) { - newBufferData[i] = originalBufferData[bufferLength - i - 1]; + if (i >= startSamples && i < endSamples) { + newBufferData[i] = originalBufferData[endSamples - counter - 1]; + counter++; + } else { + newBufferData[i] = originalBufferData[i]; + } } this.buffer = newBuffer; } else { @@ -63,7 +98,6 @@ class AudioEffects { this.source = this.audioContext.createBufferSource(); this.source.buffer = this.buffer; - this.source.playbackRate.value = playbackRate; this.name = name; } process (done) { @@ -71,17 +105,38 @@ class AudioEffects { let input; let output; switch (this.name) { + case effectTypes.FASTER: + case effectTypes.SLOWER: + this.source.playbackRate.setValueAtTime(this.playbackRate, this.adjustedTrimStartSeconds); + this.source.playbackRate.setValueAtTime(1.0, this.adjustedTrimEndSeconds); + break; case effectTypes.LOUDER: - ({input, output} = new VolumeEffect(this.audioContext, 1.25)); + ({input, output} = new VolumeEffect(this.audioContext, 1.25, + this.adjustedTrimStartSeconds, this.adjustedTrimEndSeconds)); break; case effectTypes.SOFTER: - ({input, output} = new VolumeEffect(this.audioContext, 0.75)); + ({input, output} = new VolumeEffect(this.audioContext, 0.75, + this.adjustedTrimStartSeconds, this.adjustedTrimEndSeconds)); break; case effectTypes.ECHO: - ({input, output} = new EchoEffect(this.audioContext, 0.25)); + ({input, output} = new EchoEffect(this.audioContext, + this.adjustedTrimStartSeconds, this.adjustedTrimEndSeconds)); break; case effectTypes.ROBOT: - ({input, output} = new RobotEffect(this.audioContext, 0.25)); + ({input, output} = new RobotEffect(this.audioContext, + this.adjustedTrimStartSeconds, this.adjustedTrimEndSeconds)); + break; + case effectTypes.FADEIN: + ({input, output} = new FadeEffect(this.audioContext, true, + this.adjustedTrimStartSeconds, this.adjustedTrimEndSeconds)); + break; + case effectTypes.FADEOUT: + ({input, output} = new FadeEffect(this.audioContext, false, + this.adjustedTrimStartSeconds, this.adjustedTrimEndSeconds)); + break; + case effectTypes.MUTE: + ({input, output} = new MuteEffect(this.audioContext, + this.adjustedTrimStartSeconds, this.adjustedTrimEndSeconds)); break; } @@ -96,7 +151,10 @@ class AudioEffects { this.source.start(); this.audioContext.startRendering(); - this.audioContext.oncomplete = done; + this.audioContext.oncomplete = ({renderedBuffer}) => { + done(renderedBuffer, this.adjustedTrimStart, this.adjustedTrimEnd); + }; + } } diff --git a/src/lib/audio/audio-util.js b/src/lib/audio/audio-util.js index 2b68fcbe8..a01c30de8 100644 --- a/src/lib/audio/audio-util.js +++ b/src/lib/audio/audio-util.js @@ -1,3 +1,5 @@ +import WavEncoder from 'wav-encoder'; + const SOUND_BYTE_LIMIT = 10 * 1000 * 1000; // 10mb const computeRMS = function (samples, scaling = 0.55) { @@ -23,8 +25,43 @@ const computeChunkedRMS = function (samples, chunkSize = 1024) { return chunkLevels; }; +const encodeAndAddSoundToVM = function (vm, samples, sampleRate, name, callback) { + WavEncoder.encode({ + sampleRate: sampleRate, + channelData: [samples] + }).then(wavBuffer => { + const vmSound = { + format: '', + dataFormat: 'wav', + rate: sampleRate, + sampleCount: samples.length + }; + + // Create an asset from the encoded .wav and get resulting md5 + const storage = vm.runtime.storage; + vmSound.asset = storage.createAsset( + storage.AssetType.Sound, + storage.DataFormat.WAV, + new Uint8Array(wavBuffer), + null, + true // generate md5 + ); + vmSound.assetId = vmSound.asset.assetId; + + // update vmSound object with md5 property + vmSound.md5 = `${vmSound.assetId}.${vmSound.dataFormat}`; + // The VM will update the sound name to a fresh name + vmSound.name = name; + + vm.addSound(vmSound).then(() => { + if (callback) callback(); + }); + }); +}; + export { computeRMS, computeChunkedRMS, + encodeAndAddSoundToVM, SOUND_BYTE_LIMIT }; diff --git a/src/lib/audio/effects/echo-effect.js b/src/lib/audio/effects/echo-effect.js index 3c7c06806..0ef4b93c2 100644 --- a/src/lib/audio/effects/echo-effect.js +++ b/src/lib/audio/effects/echo-effect.js @@ -1,15 +1,23 @@ class EchoEffect { - constructor (audioContext, delayTime) { + static get DELAY_TIME () { + return 0.25; + } + static get TAIL_SECONDS () { + return 0.75; + } + constructor (audioContext, startTime, endTime) { this.audioContext = audioContext; - this.delayTime = delayTime; this.input = this.audioContext.createGain(); this.output = this.audioContext.createGain(); this.effectInput = this.audioContext.createGain(); - this.effectInput.gain.value = 0.75; + this.effectInput.gain.value = 0; + + this.effectInput.gain.setValueAtTime(0.75, startTime); + this.effectInput.gain.setValueAtTime(0, endTime); this.delay = this.audioContext.createDelay(1); - this.delay.delayTime.value = delayTime; + this.delay.delayTime.value = EchoEffect.DELAY_TIME; this.decay = this.audioContext.createGain(); this.decay.gain.value = 0.3; diff --git a/src/lib/audio/effects/fade-effect.js b/src/lib/audio/effects/fade-effect.js new file mode 100644 index 000000000..5a13c5c16 --- /dev/null +++ b/src/lib/audio/effects/fade-effect.js @@ -0,0 +1,27 @@ +class FadeEffect { + constructor (audioContext, fadeIn, startSeconds, endSeconds) { + this.audioContext = audioContext; + + this.input = this.audioContext.createGain(); + this.output = this.audioContext.createGain(); + + this.gain = this.audioContext.createGain(); + + this.gain.gain.setValueAtTime(1, 0); + + if (fadeIn) { + this.gain.gain.setValueAtTime(0, startSeconds); + this.gain.gain.linearRampToValueAtTime(1, endSeconds); + } else { + this.gain.gain.setValueAtTime(1, startSeconds); + this.gain.gain.linearRampToValueAtTime(0, endSeconds); + } + + this.gain.gain.setValueAtTime(1, endSeconds); + + this.input.connect(this.gain); + this.gain.connect(this.output); + } +} + +export default FadeEffect; diff --git a/src/lib/audio/effects/mute-effect.js b/src/lib/audio/effects/mute-effect.js new file mode 100644 index 000000000..c2a2e68d5 --- /dev/null +++ b/src/lib/audio/effects/mute-effect.js @@ -0,0 +1,22 @@ +class MuteEffect { + constructor (audioContext, startSeconds, endSeconds) { + this.audioContext = audioContext; + + this.input = this.audioContext.createGain(); + this.output = this.audioContext.createGain(); + + this.gain = this.audioContext.createGain(); + + // Smoothly ramp the gain down before the start time, and up after the end time. + this.rampLength = 0.001; + this.gain.gain.setValueAtTime(1.0, Math.max(0, startSeconds - this.rampLength)); + this.gain.gain.linearRampToValueAtTime(0, startSeconds); + this.gain.gain.setValueAtTime(0, endSeconds); + this.gain.gain.linearRampToValueAtTime(1.0, endSeconds + this.rampLength); + + this.input.connect(this.gain); + this.gain.connect(this.output); + } +} + +export default MuteEffect; diff --git a/src/lib/audio/effects/robot-effect.js b/src/lib/audio/effects/robot-effect.js index 472668d53..653276eab 100644 --- a/src/lib/audio/effects/robot-effect.js +++ b/src/lib/audio/effects/robot-effect.js @@ -1,9 +1,20 @@ class RobotEffect { - constructor (audioContext) { + constructor (audioContext, startTime, endTime) { this.audioContext = audioContext; this.input = this.audioContext.createGain(); this.output = this.audioContext.createGain(); + this.passthrough = this.audioContext.createGain(); + this.effectInput = this.audioContext.createGain(); + + this.passthrough.gain.value = 1; + this.effectInput.gain.value = 0; + + this.passthrough.gain.setValueAtTime(0, startTime); + this.passthrough.gain.setValueAtTime(1, endTime); + + this.effectInput.gain.setValueAtTime(1, startTime); + this.effectInput.gain.setValueAtTime(0, endTime); // Ring modulator inspired by BBC Dalek voice // http://recherche.ircam.fr/pub/dafx11/Papers/66_e.pdf @@ -70,8 +81,13 @@ class RobotEffect { biquadFilter.frequency.value = 1000; biquadFilter.gain.value = 1.25; - this.input.connect(vcInverter1); - this.input.connect(vcDiode4); + this.input.connect(this.effectInput); + this.input.connect(this.passthrough); + + this.passthrough.connect(this.output); + + this.effectInput.connect(vcInverter1); + this.effectInput.connect(vcDiode4); vcInverter1.connect(vcDiode3); @@ -91,7 +107,7 @@ class RobotEffect { vcDiode3.connect(compressor); vcDiode4.connect(compressor); - this.input.connect(biquadFilter); + this.effectInput.connect(biquadFilter); biquadFilter.connect(compressor); compressor.connect(this.output); diff --git a/src/lib/audio/effects/volume-effect.js b/src/lib/audio/effects/volume-effect.js index bffc422ac..0cfcd6415 100644 --- a/src/lib/audio/effects/volume-effect.js +++ b/src/lib/audio/effects/volume-effect.js @@ -1,12 +1,18 @@ class VolumeEffect { - constructor (audioContext, volume) { + constructor (audioContext, volume, startSeconds, endSeconds) { this.audioContext = audioContext; this.input = this.audioContext.createGain(); this.output = this.audioContext.createGain(); this.gain = this.audioContext.createGain(); - this.gain.gain.value = volume; + + // Smoothly ramp the gain up before the start time, and down after the end time. + this.rampLength = 0.01; + this.gain.gain.setValueAtTime(1.0, Math.max(0, startSeconds - this.rampLength)); + this.gain.gain.exponentialRampToValueAtTime(volume, startSeconds); + this.gain.gain.setValueAtTime(volume, endSeconds); + this.gain.gain.exponentialRampToValueAtTime(1.0, endSeconds + this.rampLength); this.input.connect(this.gain); this.gain.connect(this.output); diff --git a/test/__mocks__/audio-effects.js b/test/__mocks__/audio-effects.js index b8a4c6994..06c36c7e8 100644 --- a/test/__mocks__/audio-effects.js +++ b/test/__mocks__/audio-effects.js @@ -14,7 +14,7 @@ export default class MockAudioEffects { this.buffer = buffer; this.name = name; this.process = jest.fn(done => { - this._finishProcessing = renderedBuffer => done({renderedBuffer}); + this._finishProcessing = renderedBuffer => done(renderedBuffer, 0, 1); }); MockAudioEffects.instance = this; } diff --git a/test/integration/sounds.test.js b/test/integration/sounds.test.js index 29ffba49f..e564f511a 100644 --- a/test/integration/sounds.test.js +++ b/test/integration/sounds.test.js @@ -1,5 +1,6 @@ import path from 'path'; import SeleniumHelper from '../helpers/selenium-helper'; +import {Key} from 'selenium-webdriver'; const { clickText, @@ -55,7 +56,6 @@ describe('Working with sounds', () => { await clickText('Faster'); await clickText('Slower'); await clickText('Robot'); - await clickText('Echo'); await clickText('Reverse'); const logs = await getLogs(); @@ -112,7 +112,7 @@ describe('Working with sounds', () => { await expect(logs).toEqual([]); }); - test.only('Adding multiple sounds at the same time', async () => { + test('Adding multiple sounds at the same time', async () => { const files = [ path.resolve(__dirname, '../fixtures/movie.wav'), path.resolve(__dirname, '../fixtures/sneaker.wav') @@ -132,4 +132,56 @@ describe('Working with sounds', () => { const logs = await getLogs(); await expect(logs).toEqual([]); }); + + test('Copy to new button adds a new sound', async () => { + await loadUri(uri); + await clickText('Sounds'); + await clickText('Copy to New', scope.soundsTab); + await clickText('Meow2', scope.soundsTab); + + const logs = await getLogs(); + await expect(logs).toEqual([]); + }); + + test('Copy and pasting within a sound changes its duration', async () => { + await loadUri(uri); + await clickText('Sounds'); + await findByText('0.85', scope.soundsTab); // Original meow sound duration + await clickText('Copy', scope.soundsTab); + await clickText('Paste', scope.soundsTab); + await findByText('1.70', scope.soundsTab); // Sound has doubled in duration + + const logs = await getLogs(); + await expect(logs).toEqual([]); + }); + + test('Can copy a sound from a sprite and paste into a sound on the stage', async () => { + await loadUri(uri); + await clickText('Sounds'); + await clickText('Copy', scope.soundsTab); // Copy the meow sound + await clickXpath('//span[text()="Stage"]'); + await findByText('0.02', scope.soundsTab); // Original pop sound duration + await clickText('Paste', scope.soundsTab); + await findByText('0.87', scope.soundsTab); // Duration of pop + meow sound + + const logs = await getLogs(); + await expect(logs).toEqual([]); + }); + + test.only('Keyboard shortcuts', async () => { + await loadUri(uri); + await clickText('Sounds'); + const el = await findByXpath('//button[@aria-label="Choose a Sound"]'); + await el.sendKeys(Key.chord(Key.COMMAND, 'a')); // Select all + await findByText('0.85', scope.soundsTab); // Meow sound duration + await el.sendKeys(Key.DELETE); + await findByText('0.00', scope.soundsTab); // Sound is now empty + await el.sendKeys(Key.chord(Key.COMMAND, 'z')); // undo + await findByText('0.85', scope.soundsTab); // Meow sound is back + await el.sendKeys(Key.chord(Key.COMMAND, Key.SHIFT, 'z')); // redo + await findByText('0.00', scope.soundsTab); // Sound is empty again + + const logs = await getLogs(); + await expect(logs).toEqual([]); + }); }); diff --git a/test/unit/components/__snapshots__/sound-editor.test.jsx.snap b/test/unit/components/__snapshots__/sound-editor.test.jsx.snap index 38ddf76c4..af7a255a4 100644 --- a/test/unit/components/__snapshots__/sound-editor.test.jsx.snap +++ b/test/unit/components/__snapshots__/sound-editor.test.jsx.snap @@ -3,6 +3,7 @@ exports[`Sound Editor Component matches snapshot 1`] = `
+
+ +
+ Copy +
+
+
+ +
+ Paste +
+
+
+ +
+ Copy to New +
+
+
+
@@ -71,9 +124,7 @@ exports[`Sound Editor Component matches snapshot 1`] = `
- - Save - + Delete
@@ -85,33 +136,26 @@ exports[`Sound Editor Component matches snapshot 1`] = ` > -
@@ -148,6 +193,73 @@ exports[`Sound Editor Component matches snapshot 1`] = ` />
+
+ +
+
+ +
+
+
-
-
-
-
- -
-
- -
-
-
@@ -381,42 +390,6 @@ exports[`Sound Editor Component matches snapshot 1`] = `
-
- -
- - Echo - -
-
-
- -
- - Robot - -
-
+
+ +
+ + Mute + +
+
+
+ +
+ + Fade in + +
+
+
+ +
+ + Fade out + +
+
+
+ +
+ + Robot + +
+
`; diff --git a/test/unit/components/sound-editor.test.jsx b/test/unit/components/sound-editor.test.jsx index efb153b1f..4b9d3805f 100644 --- a/test/unit/components/sound-editor.test.jsx +++ b/test/unit/components/sound-editor.test.jsx @@ -13,8 +13,8 @@ describe('Sound Editor Component', () => { playhead: 0.5, trimStart: 0.2, trimEnd: 0.8, - onActivateTrim: jest.fn(), onChangeName: jest.fn(), + onDelete: jest.fn(), onPlay: jest.fn(), onRedo: jest.fn(), onReverse: jest.fn(), @@ -36,19 +36,7 @@ describe('Sound Editor Component', () => { expect(component.toJSON()).toMatchSnapshot(); }); - test('trim button appears when trims are null', () => { - const wrapper = mountWithIntl( - - ); - wrapper.find('[children="Trim"]').simulate('click'); - expect(props.onActivateTrim).toHaveBeenCalled(); - }); - - test('save button appears when trims are not null', () => { + test('delete button appears when selection is not null', () => { const wrapper = mountWithIntl( { trimStart={0.25} /> ); - wrapper.find('[children="Save"]').simulate('click'); - expect(props.onActivateTrim).toHaveBeenCalled(); + wrapper.find('[children="Delete"]').simulate('click'); + expect(props.onDelete).toHaveBeenCalled(); }); test('play button appears when playhead is null', () => { @@ -100,9 +88,6 @@ describe('Sound Editor Component', () => { wrapper.find('[children="Reverse"]').simulate('click'); expect(props.onReverse).toHaveBeenCalled(); - wrapper.find('[children="Echo"]').simulate('click'); - expect(props.onEcho).toHaveBeenCalled(); - wrapper.find('[children="Robot"]').simulate('click'); expect(props.onRobot).toHaveBeenCalled(); diff --git a/test/unit/containers/sound-editor.test.jsx b/test/unit/containers/sound-editor.test.jsx index 6070b7592..b0a1a8399 100644 --- a/test/unit/containers/sound-editor.test.jsx +++ b/test/unit/containers/sound-editor.test.jsx @@ -85,29 +85,6 @@ describe('Sound Editor Container', () => { expect(component.props().playhead).toEqual(null); }); - test('it sets the component props for trimming and submits to the vm', () => { - const wrapper = mountWithIntl( - - ); - let component = wrapper.find(SoundEditorComponent); - - component.props().onActivateTrim(); - wrapper.update(); - component = wrapper.find(SoundEditorComponent); - expect(component.props().trimStart).not.toEqual(null); - expect(component.props().trimEnd).not.toEqual(null); - - component.props().onActivateTrim(); - wrapper.update(); - component = wrapper.find(SoundEditorComponent); - expect(vm.updateSoundBuffer).toHaveBeenCalled(); - expect(component.props().trimStart).toEqual(null); - expect(component.props().trimEnd).toEqual(null); - }); - test('it submits name changes to the vm', () => { const wrapper = mountWithIntl( { expect(component.prop('canRedo')).toEqual(false); // Submitting new samples should make it possible to undo - component.props().onActivateTrim(); // Activate trimming - component.props().onActivateTrim(); // Submit new samples by calling again + component.props().onFaster(); + mockAudioEffects.instance._finishProcessing(soundBuffer); wrapper.update(); component = wrapper.find(SoundEditorComponent); expect(component.prop('canUndo')).toEqual(true); @@ -264,8 +241,8 @@ describe('Sound Editor Container', () => { wrapper.update(); component = wrapper.find(SoundEditorComponent); expect(component.prop('canRedo')).toEqual(true); - component.props().onActivateTrim(); // Activate trimming - component.props().onActivateTrim(); // Submit new samples by calling again + component.props().onFaster(); + mockAudioEffects.instance._finishProcessing(soundBuffer); wrapper.update(); component = wrapper.find(SoundEditorComponent); @@ -282,8 +259,8 @@ describe('Sound Editor Container', () => { let component = wrapper.find(SoundEditorComponent); // Set up an undoable state - component.props().onActivateTrim(); // Activate trimming - component.props().onActivateTrim(); // Submit new samples by calling again + component.props().onFaster(); + mockAudioEffects.instance._finishProcessing(soundBuffer); wrapper.update(); component = wrapper.find(SoundEditorComponent); diff --git a/test/unit/util/audio-effects.test.js b/test/unit/util/audio-effects.test.js index d1057de53..330fe12a1 100644 --- a/test/unit/util/audio-effects.test.js +++ b/test/unit/util/audio-effects.test.js @@ -14,25 +14,58 @@ describe('Audio Effects manager', () => { const audioBuffer = audioContext.createBuffer(1, 400, 44100); test('changes buffer length and playback rate for faster effect', () => { - const audioEffects = new AudioEffects(audioBuffer, 'faster'); + const audioEffects = new AudioEffects(audioBuffer, 'faster', 0, 1); expect(audioEffects.audioContext._.length).toBeLessThan(400); - expect(audioEffects.source.playbackRate.value).toBeGreaterThan(1); }); test('changes buffer length and playback rate for slower effect', () => { - const audioEffects = new AudioEffects(audioBuffer, 'slower'); + const audioEffects = new AudioEffects(audioBuffer, 'slower', 0, 1); expect(audioEffects.audioContext._.length).toBeGreaterThan(400); - expect(audioEffects.source.playbackRate.value).toBeLessThan(1); }); test('changes buffer length for echo effect', () => { - const audioEffects = new AudioEffects(audioBuffer, 'echo'); + const audioEffects = new AudioEffects(audioBuffer, 'echo', 0, 1); expect(audioEffects.audioContext._.length).toBeGreaterThan(400); }); + test('updates the trim positions after an effect has changed the length of selection', () => { + const slowerEffect = new AudioEffects(audioBuffer, 'slower', 0.25, 0.75); + expect(slowerEffect.adjustedTrimStartSeconds).toEqual(slowerEffect.trimStartSeconds); + expect(slowerEffect.adjustedTrimEndSeconds).toBeGreaterThan(slowerEffect.trimEndSeconds); + + const fasterEffect = new AudioEffects(audioBuffer, 'faster', 0.25, 0.75); + expect(fasterEffect.adjustedTrimStartSeconds).toEqual(fasterEffect.trimStartSeconds); + expect(fasterEffect.adjustedTrimEndSeconds).toBeLessThan(fasterEffect.trimEndSeconds); + + // Some effects do not change the length of the selection + const fadeEffect = new AudioEffects(audioBuffer, 'fade in', 0.25, 0.75); + expect(fadeEffect.adjustedTrimStartSeconds).toEqual(fadeEffect.trimStartSeconds); + // Should be within one millisecond (flooring can change the duration by one sample) + expect(fadeEffect.adjustedTrimEndSeconds).toBeCloseTo(fadeEffect.trimEndSeconds, 3); + }); + test.skip('process starts the offline rendering context and returns a promise', () => { // @todo haven't been able to get web audio test api to actually run render }); + + test('reverse effect strictly reverses the samples', () => { + const fakeSound = [1, 2, 3, 4, 5, 6, 7, 8]; + + const fakeBuffer = audioContext.createBuffer(1, 8, 44100); + const bufferData = fakeBuffer.getChannelData(0); + fakeSound.forEach((sample, index) => { + bufferData[index] = sample; + }); + + // Reverse the entire sound + const reverseAll = new AudioEffects(fakeBuffer, 'reverse', 0, 1); + expect(Array.from(reverseAll.buffer.getChannelData(0))).toEqual(fakeSound.reverse()); + + // Reverse part of the sound + const reverseSelection = new AudioEffects(fakeBuffer, 'reverse', 0.25, 0.75); + const selectionReversed = [1, 2, 6, 5, 4, 3, 7, 8]; + expect(Array.from(reverseSelection.buffer.getChannelData(0))).toEqual(selectionReversed); + }); }); describe('Effects', () => { @@ -43,15 +76,15 @@ describe('Effects', () => { }); test('all effects provide an input and output that are connected', () => { - const robotEffect = new RobotEffect(audioContext, 0.5); + const robotEffect = new RobotEffect(audioContext, 0, 1); expect(robotEffect.input).toBeInstanceOf(AudioNode); expect(robotEffect.output).toBeInstanceOf(AudioNode); - const echoEffect = new EchoEffect(audioContext, 0.5); + const echoEffect = new EchoEffect(audioContext, 0, 1); expect(echoEffect.input).toBeInstanceOf(AudioNode); expect(echoEffect.output).toBeInstanceOf(AudioNode); - const volumeEffect = new VolumeEffect(audioContext, 0.5); + const volumeEffect = new VolumeEffect(audioContext, 0.5, 0, 1); expect(volumeEffect.input).toBeInstanceOf(AudioNode); expect(volumeEffect.output).toBeInstanceOf(AudioNode); });