diff --git a/src/containers/sound-editor.jsx b/src/containers/sound-editor.jsx index 3089b3dc3..94fca9524 100644 --- a/src/containers/sound-editor.jsx +++ b/src/containers/sound-editor.jsx @@ -6,12 +6,7 @@ import VM from 'scratch-vm'; import {connect} from 'react-redux'; -import { - computeChunkedRMS, - encodeAndAddSoundToVM, - downsampleIfNeeded, - dropEveryOtherSample -} from '../lib/audio/audio-util.js'; +import {computeChunkedRMS, encodeAndAddSoundToVM, SOUND_BYTE_LIMIT} from '../lib/audio/audio-util.js'; import AudioEffects from '../lib/audio/audio-effects.js'; import SoundEditorComponent from '../components/sound-editor/sound-editor.jsx'; import AudioBufferPlayer from '../lib/audio/audio-buffer-player.js'; @@ -44,8 +39,7 @@ class SoundEditor extends React.Component { 'paste', 'handleKeyPress', 'handleContainerClick', - 'setRef', - 'resampleBufferToRate' + 'setRef' ]); this.state = { copyBuffer: null, @@ -138,32 +132,45 @@ class SoundEditor extends React.Component { }); } submitNewSamples (samples, sampleRate, skipUndo) { - return downsampleIfNeeded({samples, sampleRate}, this.resampleBufferToRate) - .then(({samples: newSamples, sampleRate: newSampleRate}) => - WavEncoder.encode({ - sampleRate: newSampleRate, - channelData: [newSamples] - }).then(wavBuffer => { - if (!skipUndo) { - this.redoStack = []; - if (this.undoStack.length >= UNDO_STACK_SIZE) { - this.undoStack.shift(); // Drop the first element off the array - } - this.undoStack.push(this.getUndoItem()); - } - this.resetState(newSamples, newSampleRate); - this.props.vm.updateSoundBuffer( - this.props.soundIndex, - this.audioBufferPlayer.buffer, - new Uint8Array(wavBuffer)); - return true; // Edit was successful - }) - ) - .catch(e => { - // Encoding failed, or the sound was too large to save so edit is rejected - log.error(`Encountered error while trying to encode sound update: ${e}`); - return false; // Edit was not applied + // Encode the new sound into a wav so that it can be stored + let wavBuffer = null; + try { + wavBuffer = WavEncoder.encode.sync({ + sampleRate: sampleRate, + channelData: [samples] }); + + if (wavBuffer.byteLength > SOUND_BYTE_LIMIT) { + // Cancel the sound update by setting to null + wavBuffer = null; + log.error(`Refusing to encode sound larger than ${SOUND_BYTE_LIMIT} bytes`); + } + } catch (e) { + // This error state is mostly for the mock sounds used during testing. + // Any incorrect sound buffer trying to get interpretd as a Wav file + // should yield this error. + // This can also happen if the sound is too be allocated in memory. + log.error(`Encountered error while trying to encode sound update: ${e}`); + } + + // Do not submit sound if it could not be encoded (i.e. if too large) + if (wavBuffer) { + if (!skipUndo) { + this.redoStack = []; + if (this.undoStack.length >= UNDO_STACK_SIZE) { + this.undoStack.shift(); // Drop the first element off the array + } + this.undoStack.push(this.getUndoItem()); + } + this.resetState(samples, sampleRate); + this.props.vm.updateSoundBuffer( + this.props.soundIndex, + this.audioBufferPlayer.buffer, + new Uint8Array(wavBuffer)); + + return true; // Update succeeded + } + return false; // Update failed } handlePlay () { this.audioBufferPlayer.stop(); @@ -202,15 +209,13 @@ class SoundEditor extends React.Component { newSamples.set(firstPart, 0); newSamples.set(secondPart, firstPart.length); } - this.submitNewSamples(newSamples, sampleRate).then(() => { - this.setState({ - trimStart: null, - trimEnd: null - }); + this.submitNewSamples(newSamples, sampleRate); + this.setState({ + trimStart: null, + trimEnd: null }); } handleDeleteInverse () { - // Delete everything outside of the trimmers const {samples, sampleRate} = this.copyCurrentBuffer(); const sampleCount = samples.length; const startIndex = Math.floor(this.state.trimStart * sampleCount); @@ -219,13 +224,10 @@ class SoundEditor extends React.Component { if (clippedSamples.length === 0) { clippedSamples = new Float32Array(1); } - this.submitNewSamples(clippedSamples, sampleRate).then(success => { - if (success) { - this.setState({ - trimStart: null, - trimEnd: null - }); - } + this.submitNewSamples(clippedSamples, sampleRate); + this.setState({ + trimStart: null, + trimEnd: null }); } handleUpdateTrim (trimStart, trimEnd) { @@ -255,15 +257,14 @@ class SoundEditor extends React.Component { effects.process((renderedBuffer, adjustedTrimStart, adjustedTrimEnd) => { const samples = renderedBuffer.getChannelData(0); const sampleRate = renderedBuffer.sampleRate; - this.submitNewSamples(samples, sampleRate).then(success => { - if (success) { - if (this.state.trimStart === null) { - this.handlePlay(); - } else { - this.setState({trimStart: adjustedTrimStart, trimEnd: adjustedTrimEnd}, this.handlePlay); - } + const success = this.submitNewSamples(samples, sampleRate); + if (success) { + if (this.state.trimStart === null) { + this.handlePlay(); + } else { + this.setState({trimStart: adjustedTrimStart, trimEnd: adjustedTrimEnd}, this.handlePlay); } - }); + } }); } tooLoud () { @@ -286,22 +287,16 @@ class SoundEditor extends React.Component { this.redoStack.push(this.getUndoItem()); const {samples, sampleRate, trimStart, trimEnd} = this.undoStack.pop(); if (samples) { - return this.submitNewSamples(samples, sampleRate, true).then(success => { - if (success) { - this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay); - } - }); + this.submitNewSamples(samples, sampleRate, true); + this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay); } } handleRedo () { const {samples, sampleRate, trimStart, trimEnd} = this.redoStack.pop(); if (samples) { this.undoStack.push(this.getUndoItem()); - return this.submitNewSamples(samples, sampleRate, true).then(success => { - if (success) { - this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay); - } - }); + this.submitNewSamples(samples, sampleRate, true); + this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay); } } handleCopy () { @@ -327,39 +322,25 @@ class SoundEditor extends React.Component { }); } resampleBufferToRate (buffer, newRate) { - return new Promise((resolve, reject) => { - const sampleRateRatio = newRate / buffer.sampleRate; - const newLength = sampleRateRatio * buffer.samples.length; - let offlineContext; - // Try to use either OfflineAudioContext or webkitOfflineAudioContext to resample - // The constructors will throw if trying to resample at an unsupported rate - // (e.g. Safari/webkitOAC does not support lower than 44khz). - try { - if (window.OfflineAudioContext) { - offlineContext = new window.OfflineAudioContext(1, newLength, newRate); - } else if (window.webkitOfflineAudioContext) { - offlineContext = new window.webkitOfflineAudioContext(1, newLength, newRate); - } - } catch { - // If no OAC available and downsampling by 2, downsample by dropping every other sample. - if (newRate === buffer.sampleRate / 2) { - return resolve(dropEveryOtherSample(buffer)); - } - return reject('Could not resample'); + return new Promise(resolve => { + if (window.OfflineAudioContext) { + const sampleRateRatio = newRate / buffer.sampleRate; + const newLength = sampleRateRatio * buffer.samples.length; + const offlineContext = new window.OfflineAudioContext(1, newLength, newRate); + const source = offlineContext.createBufferSource(); + const audioBuffer = offlineContext.createBuffer(1, buffer.samples.length, buffer.sampleRate); + audioBuffer.getChannelData(0).set(buffer.samples); + source.buffer = audioBuffer; + source.connect(offlineContext.destination); + source.start(); + offlineContext.startRendering(); + offlineContext.oncomplete = ({renderedBuffer}) => { + resolve({ + samples: renderedBuffer.getChannelData(0), + sampleRate: newRate + }); + }; } - const source = offlineContext.createBufferSource(); - const audioBuffer = offlineContext.createBuffer(1, buffer.samples.length, buffer.sampleRate); - audioBuffer.getChannelData(0).set(buffer.samples); - source.buffer = audioBuffer; - source.connect(offlineContext.destination); - source.start(); - offlineContext.startRendering(); - offlineContext.oncomplete = ({renderedBuffer}) => { - resolve({ - samples: renderedBuffer.getChannelData(0), - sampleRate: newRate - }); - }; }); } paste () { @@ -370,11 +351,8 @@ class SoundEditor extends React.Component { const newSamples = new Float32Array(newLength); newSamples.set(samples, 0); newSamples.set(this.state.copyBuffer.samples, samples.length); - this.submitNewSamples(newSamples, this.props.sampleRate, false).then(success => { - if (success) { - this.handlePlay(); - } - }); + this.submitNewSamples(newSamples, this.props.sampleRate, false); + this.handlePlay(); } else { // else replace the selection with the pasted sound const trimStartSamples = this.state.trimStart * samples.length; @@ -393,14 +371,11 @@ class SoundEditor extends React.Component { const newDurationSeconds = newSamples.length / this.state.copyBuffer.sampleRate; const adjustedTrimStart = trimStartSeconds / newDurationSeconds; const adjustedTrimEnd = trimEndSeconds / newDurationSeconds; - this.submitNewSamples(newSamples, this.props.sampleRate, false).then(success => { - if (success) { - this.setState({ - trimStart: adjustedTrimStart, - trimEnd: adjustedTrimEnd - }, this.handlePlay); - } - }); + this.submitNewSamples(newSamples, this.props.sampleRate, false); + this.setState({ + trimStart: adjustedTrimStart, + trimEnd: adjustedTrimEnd + }, this.handlePlay); } } handlePaste () { diff --git a/src/lib/audio/audio-util.js b/src/lib/audio/audio-util.js index fba6be96b..a01c30de8 100644 --- a/src/lib/audio/audio-util.js +++ b/src/lib/audio/audio-util.js @@ -59,57 +59,9 @@ const encodeAndAddSoundToVM = function (vm, samples, sampleRate, name, callback) }); }; -/** - @typedef SoundBuffer - @type {Object} - @property {Float32Array} samples Array of audio samples - @property {number} sampleRate Audio sample rate - */ - -/** - * Downsample the given buffer to try to reduce file size below SOUND_BYTE_LIMIT - * @param {SoundBuffer} buffer - Buffer to resample - * @param {function(SoundBuffer):Promise} resampler - resampler function - * @returns {SoundBuffer} Downsampled buffer with half the sample rate - */ -const downsampleIfNeeded = (buffer, resampler) => { - const {samples, sampleRate} = buffer; - const duration = samples.length / sampleRate; - const encodedByteLength = samples.length * 2; /* bitDepth 16 bit */ - // Resolve immediately if already within byte limit - if (encodedByteLength < SOUND_BYTE_LIMIT) { - return Promise.resolve({samples, sampleRate}); - } - // If encodeable at 22khz, resample and call submitNewSamples again - if (duration * 22050 * 2 < SOUND_BYTE_LIMIT) { - return resampler({samples, sampleRate}, 22050); - } - // Cannot save this sound at 22khz, refuse to edit - // In the future we could introduce further compression here - return Promise.reject('Sound too large to save, refusing to edit'); -}; - -/** - * Drop every other sample of an audio buffer as a last-resort way of downsampling. - * @param {SoundBuffer} buffer - Buffer to resample - * @returns {SoundBuffer} Downsampled buffer with half the sample rate - */ -const dropEveryOtherSample = buffer => { - const newLength = Math.floor(buffer.samples.length / 2); - const newSamples = new Float32Array(newLength); - for (let i = 0; i < newLength; i++) { - newSamples[i] = buffer.samples[i * 2]; - } - return { - samples: newSamples, - sampleRate: buffer.rate / 2 - }; -}; - export { computeRMS, computeChunkedRMS, encodeAndAddSoundToVM, - downsampleIfNeeded, - dropEveryOtherSample + SOUND_BYTE_LIMIT }; diff --git a/test/__mocks__/audio-effects.js b/test/__mocks__/audio-effects.js index 9597eec1b..06c36c7e8 100644 --- a/test/__mocks__/audio-effects.js +++ b/test/__mocks__/audio-effects.js @@ -14,10 +14,7 @@ export default class MockAudioEffects { this.buffer = buffer; this.name = name; this.process = jest.fn(done => { - this._finishProcessing = renderedBuffer => { - done(renderedBuffer, 0, 1); - return new Promise(resolve => setTimeout(resolve)); - }; + this._finishProcessing = renderedBuffer => done(renderedBuffer, 0, 1); }); MockAudioEffects.instance = this; } diff --git a/test/unit/containers/sound-editor.test.jsx b/test/unit/containers/sound-editor.test.jsx index 4e4c967ec..b0a1a8399 100644 --- a/test/unit/containers/sound-editor.test.jsx +++ b/test/unit/containers/sound-editor.test.jsx @@ -97,7 +97,7 @@ describe('Sound Editor Container', () => { expect(vm.renameSound).toHaveBeenCalledWith(soundIndex, 'hello'); }); - test('it handles an effect by submitting the result and playing', async () => { + test('it handles an effect by submitting the result and playing', () => { const wrapper = mountWithIntl( { ); const component = wrapper.find(SoundEditorComponent); component.props().onReverse(); // Could be any of the effects, just testing the end result - await mockAudioEffects.instance._finishProcessing(soundBuffer); + mockAudioEffects.instance._finishProcessing(soundBuffer); expect(mockAudioBufferPlayer.instance.play).toHaveBeenCalled(); expect(vm.updateSoundBuffer).toHaveBeenCalled(); }); @@ -202,7 +202,7 @@ describe('Sound Editor Container', () => { expect(mockAudioEffects.instance.process).toHaveBeenCalled(); }); - test('undo/redo stack state', async () => { + test('undo/redo stack state', () => { const wrapper = mountWithIntl( { // Submitting new samples should make it possible to undo component.props().onFaster(); - await mockAudioEffects.instance._finishProcessing(soundBuffer); + mockAudioEffects.instance._finishProcessing(soundBuffer); wrapper.update(); component = wrapper.find(SoundEditorComponent); expect(component.prop('canUndo')).toEqual(true); expect(component.prop('canRedo')).toEqual(false); // Undoing should make it possible to redo and not possible to undo again - await component.props().onUndo(); + component.props().onUndo(); wrapper.update(); component = wrapper.find(SoundEditorComponent); expect(component.prop('canUndo')).toEqual(false); expect(component.prop('canRedo')).toEqual(true); // Redoing should make it possible to undo and not possible to redo again - await component.props().onRedo(); + component.props().onRedo(); wrapper.update(); component = wrapper.find(SoundEditorComponent); expect(component.prop('canUndo')).toEqual(true); expect(component.prop('canRedo')).toEqual(false); // New submission should clear the redo stack - await component.props().onUndo(); // Undo to go back to a state where redo is enabled + component.props().onUndo(); // Undo to go back to a state where redo is enabled wrapper.update(); component = wrapper.find(SoundEditorComponent); expect(component.prop('canRedo')).toEqual(true); component.props().onFaster(); - await mockAudioEffects.instance._finishProcessing(soundBuffer); + mockAudioEffects.instance._finishProcessing(soundBuffer); wrapper.update(); component = wrapper.find(SoundEditorComponent); expect(component.prop('canRedo')).toEqual(false); }); - test('undo and redo submit new samples and play the sound', async () => { + test('undo and redo submit new samples and play the sound', () => { const wrapper = mountWithIntl( { // Set up an undoable state component.props().onFaster(); - await mockAudioEffects.instance._finishProcessing(soundBuffer); + mockAudioEffects.instance._finishProcessing(soundBuffer); wrapper.update(); component = wrapper.find(SoundEditorComponent); // Undo should update the sound buffer and play the new samples - await component.props().onUndo(); + component.props().onUndo(); expect(mockAudioBufferPlayer.instance.play).toHaveBeenCalled(); expect(vm.updateSoundBuffer).toHaveBeenCalled(); @@ -274,7 +274,7 @@ describe('Sound Editor Container', () => { mockAudioBufferPlayer.instance.play.mockClear(); // Undo should update the sound buffer and play the new samples - await component.props().onRedo(); + component.props().onRedo(); expect(mockAudioBufferPlayer.instance.play).toHaveBeenCalled(); expect(vm.updateSoundBuffer).toHaveBeenCalled(); }); diff --git a/test/unit/util/audio-util.test.js b/test/unit/util/audio-util.test.js index 79ff55e8b..24ddcd455 100644 --- a/test/unit/util/audio-util.test.js +++ b/test/unit/util/audio-util.test.js @@ -1,9 +1,4 @@ -import { - computeRMS, - computeChunkedRMS, - downsampleIfNeeded, - dropEveryOtherSample -} from '../../../src/lib/audio/audio-util'; +import {computeRMS, computeChunkedRMS} from '../../../src/lib/audio/audio-util'; describe('computeRMS', () => { test('returns 0 when given no samples', () => { @@ -54,44 +49,3 @@ describe('computeChunkedRMS', () => { expect(chunkedLevels).toEqual([Math.sqrt(1 / 0.55), Math.sqrt(1 / 0.55)]); }); }); - -describe('downsampleIfNeeded', () => { - const samples = {length: 1}; - const sampleRate = 44100; - test('returns given data when no downsampling needed', async () => { - samples.length = 1; - const res = await downsampleIfNeeded({samples, sampleRate}, null); - expect(res.samples).toEqual(samples); - expect(res.sampleRate).toEqual(sampleRate); - }); - test('downsamples to 22050 if that puts it under the limit', async () => { - samples.length = 44100 * 3 * 60; - const resampler = jest.fn(() => 'TEST'); - const res = await downsampleIfNeeded({samples, sampleRate}, resampler); - expect(resampler).toHaveBeenCalledWith({samples, sampleRate}, 22050); - expect(res).toEqual('TEST'); - }); - test('fails if resampling would not put it under the limit', async () => { - samples.length = 44100 * 4 * 60; - try { - await downsampleIfNeeded({samples, sampleRate}, null); - } catch (e) { - expect(e).toEqual('Sound too large to save, refusing to edit'); - } - }); -}); - -describe('dropEveryOtherSample', () => { - const buffer = { - samples: [1, 0, 2, 0, 3, 0], - sampleRate: 2 - }; - test('result is half the length', () => { - const {samples} = dropEveryOtherSample(buffer, 1); - expect(samples.length).toEqual(Math.floor(buffer.samples.length / 2)); - }); - test('result contains only even-index items', () => { - const {samples} = dropEveryOtherSample(buffer, 1); - expect(samples).toEqual(new Float32Array([1, 2, 3])); - }); -});