ScCN/scratch-gui
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #5376 from LLK/revert-5338-fix-long-sound

Browse Source 
Revert "Implement sound downsampler to try to allow more sounds to be edited"
This commit is contained in:
Paul Kaplan authored and GitHub committed 2020-01-09 11:47:45 -05:00
2 parents e476548c64 79277a1227
commit 1dcba6ecd5
5 files changed
+99 -221
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/containers/sound-editor.jsx
+84 -109
View File
@@ -6,12 +6,7 @@ import VM from 'scratch-vm';
import {connect} from 'react-redux';
import {
computeChunkedRMS,
encodeAndAddSoundToVM,
downsampleIfNeeded,
dropEveryOtherSample
} from '../lib/audio/audio-util.js';
import {computeChunkedRMS, encodeAndAddSoundToVM, SOUND_BYTE_LIMIT} from '../lib/audio/audio-util.js';
import AudioEffects from '../lib/audio/audio-effects.js';
import SoundEditorComponent from '../components/sound-editor/sound-editor.jsx';
import AudioBufferPlayer from '../lib/audio/audio-buffer-player.js';
@@ -44,8 +39,7 @@ class SoundEditor extends React.Component {
'paste',
'handleKeyPress',
'handleContainerClick',
'setRef',
'resampleBufferToRate'
'setRef'
]);
this.state = {
copyBuffer: null,
@@ -138,32 +132,45 @@ class SoundEditor extends React.Component {
});
}
submitNewSamples (samples, sampleRate, skipUndo) {
return downsampleIfNeeded({samples, sampleRate}, this.resampleBufferToRate)
.then(({samples: newSamples, sampleRate: newSampleRate}) =>
WavEncoder.encode({
sampleRate: newSampleRate,
channelData: [newSamples]
}).then(wavBuffer => {
if (!skipUndo) {
this.redoStack = [];
if (this.undoStack.length >= UNDO_STACK_SIZE) {
this.undoStack.shift(); // Drop the first element off the array
}
this.undoStack.push(this.getUndoItem());
}
this.resetState(newSamples, newSampleRate);
this.props.vm.updateSoundBuffer(
this.props.soundIndex,
this.audioBufferPlayer.buffer,
new Uint8Array(wavBuffer));
return true; // Edit was successful
})
)
.catch(e => {
// Encoding failed, or the sound was too large to save so edit is rejected
log.error(`Encountered error while trying to encode sound update: ${e}`);
return false; // Edit was not applied
// Encode the new sound into a wav so that it can be stored
let wavBuffer = null;
try {
wavBuffer = WavEncoder.encode.sync({
sampleRate: sampleRate,
channelData: [samples]
});
if (wavBuffer.byteLength > SOUND_BYTE_LIMIT) {
// Cancel the sound update by setting to null
wavBuffer = null;
log.error(`Refusing to encode sound larger than ${SOUND_BYTE_LIMIT} bytes`);
}
} catch (e) {
// This error state is mostly for the mock sounds used during testing.
// Any incorrect sound buffer trying to get interpretd as a Wav file
// should yield this error.
// This can also happen if the sound is too be allocated in memory.
log.error(`Encountered error while trying to encode sound update: ${e}`);
}
// Do not submit sound if it could not be encoded (i.e. if too large)
if (wavBuffer) {
if (!skipUndo) {
this.redoStack = [];
if (this.undoStack.length >= UNDO_STACK_SIZE) {
this.undoStack.shift(); // Drop the first element off the array
}
this.undoStack.push(this.getUndoItem());
}
this.resetState(samples, sampleRate);
this.props.vm.updateSoundBuffer(
this.props.soundIndex,
this.audioBufferPlayer.buffer,
new Uint8Array(wavBuffer));
return true; // Update succeeded
}
return false; // Update failed
}
handlePlay () {
this.audioBufferPlayer.stop();
@@ -202,15 +209,13 @@ class SoundEditor extends React.Component {
newSamples.set(firstPart, 0);
newSamples.set(secondPart, firstPart.length);
}
this.submitNewSamples(newSamples, sampleRate).then(() => {
this.setState({
trimStart: null,
trimEnd: null
});
this.submitNewSamples(newSamples, sampleRate);
this.setState({
trimStart: null,
trimEnd: null
});
}
handleDeleteInverse () {
// Delete everything outside of the trimmers
const {samples, sampleRate} = this.copyCurrentBuffer();
const sampleCount = samples.length;
const startIndex = Math.floor(this.state.trimStart * sampleCount);
@@ -219,13 +224,10 @@ class SoundEditor extends React.Component {
if (clippedSamples.length === 0) {
clippedSamples = new Float32Array(1);
}
this.submitNewSamples(clippedSamples, sampleRate).then(success => {
if (success) {
this.setState({
trimStart: null,
trimEnd: null
});
}
this.submitNewSamples(clippedSamples, sampleRate);
this.setState({
trimStart: null,
trimEnd: null
});
}
handleUpdateTrim (trimStart, trimEnd) {
@@ -255,15 +257,14 @@ class SoundEditor extends React.Component {
effects.process((renderedBuffer, adjustedTrimStart, adjustedTrimEnd) => {
const samples = renderedBuffer.getChannelData(0);
const sampleRate = renderedBuffer.sampleRate;
this.submitNewSamples(samples, sampleRate).then(success => {
if (success) {
if (this.state.trimStart === null) {
this.handlePlay();
} else {
this.setState({trimStart: adjustedTrimStart, trimEnd: adjustedTrimEnd}, this.handlePlay);
}
const success = this.submitNewSamples(samples, sampleRate);
if (success) {
if (this.state.trimStart === null) {
this.handlePlay();
} else {
this.setState({trimStart: adjustedTrimStart, trimEnd: adjustedTrimEnd}, this.handlePlay);
}
});
}
});
}
tooLoud () {
@@ -286,22 +287,16 @@ class SoundEditor extends React.Component {
this.redoStack.push(this.getUndoItem());
const {samples, sampleRate, trimStart, trimEnd} = this.undoStack.pop();
if (samples) {
return this.submitNewSamples(samples, sampleRate, true).then(success => {
if (success) {
this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay);
}
});
this.submitNewSamples(samples, sampleRate, true);
this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay);
}
}
handleRedo () {
const {samples, sampleRate, trimStart, trimEnd} = this.redoStack.pop();
if (samples) {
this.undoStack.push(this.getUndoItem());
return this.submitNewSamples(samples, sampleRate, true).then(success => {
if (success) {
this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay);
}
});
this.submitNewSamples(samples, sampleRate, true);
this.setState({trimStart: trimStart, trimEnd: trimEnd}, this.handlePlay);
}
}
handleCopy () {
@@ -327,39 +322,25 @@ class SoundEditor extends React.Component {
});
}
resampleBufferToRate (buffer, newRate) {
return new Promise((resolve, reject) => {
const sampleRateRatio = newRate / buffer.sampleRate;
const newLength = sampleRateRatio * buffer.samples.length;
let offlineContext;
// Try to use either OfflineAudioContext or webkitOfflineAudioContext to resample
// The constructors will throw if trying to resample at an unsupported rate
// (e.g. Safari/webkitOAC does not support lower than 44khz).
try {
if (window.OfflineAudioContext) {
offlineContext = new window.OfflineAudioContext(1, newLength, newRate);
} else if (window.webkitOfflineAudioContext) {
offlineContext = new window.webkitOfflineAudioContext(1, newLength, newRate);
}
} catch {
// If no OAC available and downsampling by 2, downsample by dropping every other sample.
if (newRate === buffer.sampleRate / 2) {
return resolve(dropEveryOtherSample(buffer));
}
return reject('Could not resample');
return new Promise(resolve => {
if (window.OfflineAudioContext) {
const sampleRateRatio = newRate / buffer.sampleRate;
const newLength = sampleRateRatio * buffer.samples.length;
const offlineContext = new window.OfflineAudioContext(1, newLength, newRate);
const source = offlineContext.createBufferSource();
const audioBuffer = offlineContext.createBuffer(1, buffer.samples.length, buffer.sampleRate);
audioBuffer.getChannelData(0).set(buffer.samples);
source.buffer = audioBuffer;
source.connect(offlineContext.destination);
source.start();
offlineContext.startRendering();
offlineContext.oncomplete = ({renderedBuffer}) => {
resolve({
samples: renderedBuffer.getChannelData(0),
sampleRate: newRate
});
};
}
const source = offlineContext.createBufferSource();
const audioBuffer = offlineContext.createBuffer(1, buffer.samples.length, buffer.sampleRate);
audioBuffer.getChannelData(0).set(buffer.samples);
source.buffer = audioBuffer;
source.connect(offlineContext.destination);
source.start();
offlineContext.startRendering();
offlineContext.oncomplete = ({renderedBuffer}) => {
resolve({
samples: renderedBuffer.getChannelData(0),
sampleRate: newRate
});
};
});
}
paste () {
@@ -370,11 +351,8 @@ class SoundEditor extends React.Component {
const newSamples = new Float32Array(newLength);
newSamples.set(samples, 0);
newSamples.set(this.state.copyBuffer.samples, samples.length);
this.submitNewSamples(newSamples, this.props.sampleRate, false).then(success => {
if (success) {
this.handlePlay();
}
});
this.submitNewSamples(newSamples, this.props.sampleRate, false);
this.handlePlay();
} else {
// else replace the selection with the pasted sound
const trimStartSamples = this.state.trimStart * samples.length;
@@ -393,14 +371,11 @@ class SoundEditor extends React.Component {
const newDurationSeconds = newSamples.length / this.state.copyBuffer.sampleRate;
const adjustedTrimStart = trimStartSeconds / newDurationSeconds;
const adjustedTrimEnd = trimEndSeconds / newDurationSeconds;
this.submitNewSamples(newSamples, this.props.sampleRate, false).then(success => {
if (success) {
this.setState({
trimStart: adjustedTrimStart,
trimEnd: adjustedTrimEnd
}, this.handlePlay);
}
});
this.submitNewSamples(newSamples, this.props.sampleRate, false);
this.setState({
trimStart: adjustedTrimStart,
trimEnd: adjustedTrimEnd
}, this.handlePlay);
}
}
handlePaste () {
src/lib/audio/audio-util.js
+1 -49
View File
@@ -59,57 +59,9 @@ const encodeAndAddSoundToVM = function (vm, samples, sampleRate, name, callback)
});
};
/**
@typedef SoundBuffer
@type {Object}
@property {Float32Array} samples Array of audio samples
@property {number} sampleRate Audio sample rate
*/
/**
* Downsample the given buffer to try to reduce file size below SOUND_BYTE_LIMIT
* @param {SoundBuffer} buffer - Buffer to resample
* @param {function(SoundBuffer):Promise<SoundBuffer>} resampler - resampler function
* @returns {SoundBuffer} Downsampled buffer with half the sample rate
*/
const downsampleIfNeeded = (buffer, resampler) => {
const {samples, sampleRate} = buffer;
const duration = samples.length / sampleRate;
const encodedByteLength = samples.length * 2; /* bitDepth 16 bit */
// Resolve immediately if already within byte limit
if (encodedByteLength < SOUND_BYTE_LIMIT) {
return Promise.resolve({samples, sampleRate});
}
// If encodeable at 22khz, resample and call submitNewSamples again
if (duration * 22050 * 2 < SOUND_BYTE_LIMIT) {
return resampler({samples, sampleRate}, 22050);
}
// Cannot save this sound at 22khz, refuse to edit
// In the future we could introduce further compression here
return Promise.reject('Sound too large to save, refusing to edit');
};
/**
* Drop every other sample of an audio buffer as a last-resort way of downsampling.
* @param {SoundBuffer} buffer - Buffer to resample
* @returns {SoundBuffer} Downsampled buffer with half the sample rate
*/
const dropEveryOtherSample = buffer => {
const newLength = Math.floor(buffer.samples.length / 2);
const newSamples = new Float32Array(newLength);
for (let i = 0; i < newLength; i++) {
newSamples[i] = buffer.samples[i * 2];
}
return {
samples: newSamples,
sampleRate: buffer.rate / 2
};
};
export {
computeRMS,
computeChunkedRMS,
encodeAndAddSoundToVM,
downsampleIfNeeded,
dropEveryOtherSample
SOUND_BYTE_LIMIT
};
test/__mocks__/audio-effects.js
+1 -4
View File
@@ -14,10 +14,7 @@ export default class MockAudioEffects {
this.buffer = buffer;
this.name = name;
this.process = jest.fn(done => {
this._finishProcessing = renderedBuffer => {
done(renderedBuffer, 0, 1);
return new Promise(resolve => setTimeout(resolve));
};
this._finishProcessing = renderedBuffer => done(renderedBuffer, 0, 1);
});
MockAudioEffects.instance = this;
}
test/unit/containers/sound-editor.test.jsx
+12 -12
View File
@@ -97,7 +97,7 @@ describe('Sound Editor Container', () => {
expect(vm.renameSound).toHaveBeenCalledWith(soundIndex, 'hello');
});
test('it handles an effect by submitting the result and playing', async () => {
test('it handles an effect by submitting the result and playing', () => {
const wrapper = mountWithIntl(
<SoundEditor
soundIndex={soundIndex}
@@ -106,7 +106,7 @@ describe('Sound Editor Container', () => {
);
const component = wrapper.find(SoundEditorComponent);
component.props().onReverse(); // Could be any of the effects, just testing the end result
await mockAudioEffects.instance._finishProcessing(soundBuffer);
mockAudioEffects.instance._finishProcessing(soundBuffer);
expect(mockAudioBufferPlayer.instance.play).toHaveBeenCalled();
expect(vm.updateSoundBuffer).toHaveBeenCalled();
});
@@ -202,7 +202,7 @@ describe('Sound Editor Container', () => {
expect(mockAudioEffects.instance.process).toHaveBeenCalled();
});
test('undo/redo stack state', async () => {
test('undo/redo stack state', () => {
const wrapper = mountWithIntl(
<SoundEditor
soundIndex={soundIndex}
@@ -216,40 +216,40 @@ describe('Sound Editor Container', () => {
// Submitting new samples should make it possible to undo
component.props().onFaster();
await mockAudioEffects.instance._finishProcessing(soundBuffer);
mockAudioEffects.instance._finishProcessing(soundBuffer);
wrapper.update();
component = wrapper.find(SoundEditorComponent);
expect(component.prop('canUndo')).toEqual(true);
expect(component.prop('canRedo')).toEqual(false);
// Undoing should make it possible to redo and not possible to undo again
await component.props().onUndo();
component.props().onUndo();
wrapper.update();
component = wrapper.find(SoundEditorComponent);
expect(component.prop('canUndo')).toEqual(false);
expect(component.prop('canRedo')).toEqual(true);
// Redoing should make it possible to undo and not possible to redo again
await component.props().onRedo();
component.props().onRedo();
wrapper.update();
component = wrapper.find(SoundEditorComponent);
expect(component.prop('canUndo')).toEqual(true);
expect(component.prop('canRedo')).toEqual(false);
// New submission should clear the redo stack
await component.props().onUndo(); // Undo to go back to a state where redo is enabled
component.props().onUndo(); // Undo to go back to a state where redo is enabled
wrapper.update();
component = wrapper.find(SoundEditorComponent);
expect(component.prop('canRedo')).toEqual(true);
component.props().onFaster();
await mockAudioEffects.instance._finishProcessing(soundBuffer);
mockAudioEffects.instance._finishProcessing(soundBuffer);
wrapper.update();
component = wrapper.find(SoundEditorComponent);
expect(component.prop('canRedo')).toEqual(false);
});
test('undo and redo submit new samples and play the sound', async () => {
test('undo and redo submit new samples and play the sound', () => {
const wrapper = mountWithIntl(
<SoundEditor
soundIndex={soundIndex}
@@ -260,12 +260,12 @@ describe('Sound Editor Container', () => {
// Set up an undoable state
component.props().onFaster();
await mockAudioEffects.instance._finishProcessing(soundBuffer);
mockAudioEffects.instance._finishProcessing(soundBuffer);
wrapper.update();
component = wrapper.find(SoundEditorComponent);
// Undo should update the sound buffer and play the new samples
await component.props().onUndo();
component.props().onUndo();
expect(mockAudioBufferPlayer.instance.play).toHaveBeenCalled();
expect(vm.updateSoundBuffer).toHaveBeenCalled();
@@ -274,7 +274,7 @@ describe('Sound Editor Container', () => {
mockAudioBufferPlayer.instance.play.mockClear();
// Undo should update the sound buffer and play the new samples
await component.props().onRedo();
component.props().onRedo();
expect(mockAudioBufferPlayer.instance.play).toHaveBeenCalled();
expect(vm.updateSoundBuffer).toHaveBeenCalled();
});
test/unit/util/audio-util.test.js
+1 -47
View File
@@ -1,9 +1,4 @@
import {
computeRMS,
computeChunkedRMS,
downsampleIfNeeded,
dropEveryOtherSample
} from '../../../src/lib/audio/audio-util';
import {computeRMS, computeChunkedRMS} from '../../../src/lib/audio/audio-util';
describe('computeRMS', () => {
test('returns 0 when given no samples', () => {
@@ -54,44 +49,3 @@ describe('computeChunkedRMS', () => {
expect(chunkedLevels).toEqual([Math.sqrt(1 / 0.55), Math.sqrt(1 / 0.55)]);
});
});
describe('downsampleIfNeeded', () => {
const samples = {length: 1};
const sampleRate = 44100;
test('returns given data when no downsampling needed', async () => {
samples.length = 1;
const res = await downsampleIfNeeded({samples, sampleRate}, null);
expect(res.samples).toEqual(samples);
expect(res.sampleRate).toEqual(sampleRate);
});
test('downsamples to 22050 if that puts it under the limit', async () => {
samples.length = 44100 * 3 * 60;
const resampler = jest.fn(() => 'TEST');
const res = await downsampleIfNeeded({samples, sampleRate}, resampler);
expect(resampler).toHaveBeenCalledWith({samples, sampleRate}, 22050);
expect(res).toEqual('TEST');
});
test('fails if resampling would not put it under the limit', async () => {
samples.length = 44100 * 4 * 60;
try {
await downsampleIfNeeded({samples, sampleRate}, null);
} catch (e) {
expect(e).toEqual('Sound too large to save, refusing to edit');
}
});
});
describe('dropEveryOtherSample', () => {
const buffer = {
samples: [1, 0, 2, 0, 3, 0],
sampleRate: 2
};
test('result is half the length', () => {
const {samples} = dropEveryOtherSample(buffer, 1);
expect(samples.length).toEqual(Math.floor(buffer.samples.length / 2));
});
test('result contains only even-index items', () => {
const {samples} = dropEveryOtherSample(buffer, 1);
expect(samples).toEqual(new Float32Array([1, 2, 3]));
});
});