From b02d7e8cd84a633fc1897be5eb4e7464b6babdbf Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Mon, 8 Jan 2018 15:34:58 -0500 Subject: [PATCH 1/5] Read benchmark warm up time and recording time from uri --- src/playground/benchmark.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/playground/benchmark.js b/src/playground/benchmark.js index 3fd1bc2a..843ffca6 100644 --- a/src/playground/benchmark.js +++ b/src/playground/benchmark.js @@ -14,7 +14,7 @@ document.querySelector('.run') }, false); const loadProject = function () { - let id = location.hash.substring(1); + let id = location.hash.substring(1).split(',')[0]; if (id.length < 1 || !isFinite(id)) { id = projectInput.value; } @@ -310,7 +310,7 @@ class ProfilerRun { dom: document.getElementsByClassName('profile-count-group')[0], runningStats, - maxRecordedTime: 6000 + maxRecordedTime }); const frames = this.frames = new Frames(profiler); @@ -388,10 +388,21 @@ window.onload = function () { .innerText = progress.complete; }).on(storage); + let warmUpTime = 4000; + let maxRecordedTime = 6000; + + if (location.hash) { + const split = location.hash.substring(1).split(','); + if (split[1] && split[1].length > 0) { + warmUpTime = Number(split[1]); + } + maxRecordedTime = Number(split[2] || '0') || 6000; + } + new ProfilerRun({ vm, - warmUpTime: 4000, - maxRecordedTime: 6000 + warmUpTime, + maxRecordedTime }).run(); // Instantiate the renderer and connect it to the VM. From c6f4d89371815627ef846b836d19dd7f0cf5a0b6 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Mon, 8 Jan 2018 15:35:43 -0500 Subject: [PATCH 2/5] Add benchmark suite, run and report on multiple benchmark projects --- src/playground/benchmark.js | 16 ++ src/playground/suite.css | 45 +++++ src/playground/suite.html | 23 +++ src/playground/suite.js | 385 ++++++++++++++++++++++++++++++++++++ 4 files changed, 469 insertions(+) create mode 100644 src/playground/suite.css create mode 100644 src/playground/suite.html create mode 100644 src/playground/suite.js diff --git a/src/playground/benchmark.js b/src/playground/benchmark.js index 843ffca6..bca3e053 100644 --- a/src/playground/benchmark.js +++ b/src/playground/benchmark.js @@ -348,11 +348,21 @@ class ProfilerRun { run () { loadProject(); + window.parent.postMessage({ + type: 'BENCH_MESSAGE_LOADING' + }, '*'); + this.vm.on('workspaceUpdate', () => { setTimeout(() => { + window.parent.postMessage({ + type: 'BENCH_MESSAGE_WARMING_UP' + }, '*'); this.vm.greenFlag(); }, 100); setTimeout(() => { + window.parent.postMessage({ + type: 'BENCH_MESSAGE_ACTIVE' + }, '*'); this.vm.runtime.profiler = this.profiler; }, 100 + this.warmUpTime); setTimeout(() => { @@ -362,6 +372,12 @@ class ProfilerRun { this.frameTable.render(); this.opcodeTable.render(); + + window.parent.postMessage({ + type: 'BENCH_MESSAGE_COMPLETE', + frames: this.frames.frames, + opcodes: this.opcodes.opcodes + }, '*'); }, 100 + this.warmUpTime + this.maxRecordedTime); }); } diff --git a/src/playground/suite.css b/src/playground/suite.css new file mode 100644 index 00000000..08eb5d61 --- /dev/null +++ b/src/playground/suite.css @@ -0,0 +1,45 @@ +html, body { + width: 100%; + height: 100%; + margin: 0; + overflow: hidden; +} + +.runner-controls { + position: absolute; + width: 30em; + height: 100%; + left: 0; + right: 30em; + top: 0; + bottom: 0; + overflow: scroll; + padding: 1em; + box-sizing: border-box; +} + +.bench-frame-owner { + position: absolute; + width: calc(100% - 30em); + height: 100%; + left: 30em; + right: 100%; + top: 0; + bottom: 0; +} + +.fixture-project { + display: inline-block; +} + +.fixture-warm-up { + display: inline-block; +} + +.fixture-recording { + display: inline-block; +} + +.result-status { + float: right; +} diff --git a/src/playground/suite.html b/src/playground/suite.html new file mode 100644 index 00000000..8acd7ca6 --- /dev/null +++ b/src/playground/suite.html @@ -0,0 +1,23 @@ + + + + + + Scratch VM Benchmark Suite + + + +
+

Scratch VM Benchmark Suite

+
+ +
+
+
+ +
+ + + + + diff --git a/src/playground/suite.js b/src/playground/suite.js new file mode 100644 index 00000000..11482812 --- /dev/null +++ b/src/playground/suite.js @@ -0,0 +1,385 @@ +const soon = (() => { + let _soon; + return () => { + if (!_soon) { + _soon = Promise.resolve() + .then(() => { + _soon = null; + }); + } + return _soon; + }; +})(); + +class Emitter { + constructor () { + Object.defineProperty(this, '_listeners', { + value: {}, + enumerable: false + }); + } + on (name, listener, context) { + if (!this._listeners[name]) { + this._listeners[name] = []; + } + + this._listeners[name].push(listener, context); + } + off (name, listener, context) { + if (this._listeners[name]) { + if (listener) { + for (let i = 0; i < this._listeners[name].length; i += 2) { + if ( + this._listeners[name][i] === listener && + this._listeners[name][i + 1] === context) { + this._listeners[name].splice(i, 2); + i -= 2; + } + } + } else { + for (let i = 0; i < this._listeners[name].length; i += 2) { + if (this._listeners[name][i + 1] === context) { + this._listeners[name].splice(i, 2); + i -= 2; + } + } + } + } + } + emit (name, ...args) { + if (this._listeners[name]) { + for (let i = 0; i < this._listeners[name].length; i += 2) { + this._listeners[name][i].call(this._listeners[name][i + 1] || this, ...args); + } + } + } +} + +class BenchFrameStream extends Emitter { + constructor (frame) { + super(); + + this.frame = frame; + window.addEventListener('message', message => { + this.emit('message', message.data); + }); + } + + send (message) { + this.frame.send(message); + } +} + +const BENCH_MESSAGE_TYPE = { + LOAD: 'BENCH_MESSAGE_LOAD', + LOADING: 'BENCH_MESSAGE_LOADING', + WARMING_UP: 'BENCH_MESSAGE_WARMING_UP', + ACTIVE: 'BENCH_MESSAGE_ACTIVE', + COMPLETE: 'BENCH_MESSAGE_COMPLETE' +}; + +class BenchUtil { + constructor (frame) { + this.frame = frame; + this.benchStream = new BenchFrameStream(frame); + } + + startBench (args) { + Promise.resolve() + .then(() => new Promise(resolve => setTimeout(resolve, 100))) + .then(() => { + this.frame.contentWindow.location.assign('about:blank'); + }) + .then(() => new Promise(resolve => setTimeout(resolve, 100))) + .then(() => { + this.frame.contentWindow.location.assign( + `/benchmark/#${args.projectId},${args.warmUpTime},${args.recordingTime}`); + }); + } +} + +const BENCH_STATUS = { + INACTIVE: 'BENCH_STATUS_INACTIVE', + STARTING: 'BENCH_STATUS_STARTING', + LOADING: 'BENCH_STATUS_LOADING', + WARMING_UP: 'BENCH_STATUS_WARMING_UP', + ACTIVE: 'BENCH_STATUS_ACTIVE', + COMPLETE: 'BENCH_STATUS_COMPLETE' +}; + +class BenchResult { + constructor ({fixture, status = BENCH_STATUS.INACTIVE, frames = null, opcodes = null}) { + this.fixture = fixture; + this.status = status; + this.frames = frames; + this.opcodes = opcodes; + } +} + +class BenchFixture extends Emitter { + constructor ({ + projectId, + warmUpTime = 4000, + recordingTime = 6000 + }) { + super(); + + this.projectId = projectId; + this.warmUpTime = warmUpTime; + this.recordingTime = recordingTime; + } + + get id () { + return `${this.projectId}-${this.warmUpTime}-${this.recordingTime}`; + } + + run (util) { + return new Promise(resolve => { + util.benchStream.on('message', message => { + const result = { + fixture: this, + status: BENCH_STATUS.STARTING, + frames: null, + opcodes: null + }; + if (message.type === BENCH_MESSAGE_TYPE.LOADING) { + result.status = BENCH_STATUS.LOADING; + } else if (message.type === BENCH_MESSAGE_TYPE.WARMING_UP) { + result.status = BENCH_STATUS.WARMING_UP; + } else if (message.type === BENCH_MESSAGE_TYPE.ACTIVE) { + result.status = BENCH_STATUS.ACTIVE; + } else if (message.type === BENCH_MESSAGE_TYPE.COMPLETE) { + result.status = BENCH_STATUS.COMPLETE; + result.frames = message.frames; + result.opcodes = message.opcodes; + resolve(new BenchResult(result)); + util.benchStream.off('message', null, this); + } + this.emit('result', new BenchResult(result)); + }, this); + util.startBench(this); + }); + } +} + +class BenchSuiteResult extends Emitter { + constructor ({suite, results = []}) { + super(); + + this.suite = suite; + this.results = results; + + if (suite) { + suite.on('result', result => { + if (result.status === BENCH_STATUS.COMPLETE) { + this.results.push(results); + this.emit('add', this); + } + }); + } + } +} + +class BenchSuite extends Emitter { + constructor (fixtures = []) { + super(); + + this.fixtures = fixtures; + } + + add (fixture) { + this.fixtures.push(fixture); + } + + run (util) { + return new Promise(resolve => { + const fixtures = this.fixtures.slice(); + const results = []; + const push = result => { + result.fixture.off('result', null, this); + results.push(result); + }; + const emitResult = this.emit.bind(this, 'result'); + const pop = () => { + const fixture = fixtures.shift(); + if (fixture) { + fixture.on('result', emitResult, this); + fixture.run(util) + .then(push) + .then(pop); + } else { + resolve(new BenchSuiteResult({suite: this, results})); + } + }; + pop(); + }); + } +} + +class BenchRunner extends Emitter { + constructor ({frame, suite}) { + super(); + + this.frame = frame; + this.suite = suite; + this.util = new BenchUtil(frame); + } + + run () { + return this.suite.run(this.util); + } +} + +const viewNames = { + [BENCH_STATUS.INACTIVE]: 'Inactive', + [BENCH_STATUS.STARTING]: 'Starting', + [BENCH_STATUS.LOADING]: 'Loading', + [BENCH_STATUS.WARMING_UP]: 'Warming Up', + [BENCH_STATUS.ACTIVE]: 'Active', + [BENCH_STATUS.COMPLETE]: 'Complete' +}; + +class BenchResultView { + constructor ({result}) { + this.result = result; + this.dom = document.createElement('div'); + } + + update (result) { + soon().then(() => this.render(result)); + } + + render (newResult = this.result) { + const blockFunctionFrame = (newResult.frames ? newResult.frames : []) + .find(frame => frame.name === 'blockFunction'); + const stepThreadsInnerFrame = (newResult.frames ? newResult.frames : []) + .find(frame => frame.name === 'Sequencer.stepThreads#inner'); + + const blocksPerSecond = blockFunctionFrame ? + (blockFunctionFrame.executions / (stepThreadsInnerFrame.totalTime / 1000)) | 0 : + 0; + + this.dom.innerHTML = ` +
+ ${newResult.fixture.projectId} +
+ (${newResult.fixture.warmUpTime / 1000}s + ${newResult.fixture.recordingTime / 1000}s) +
+ ${blockFunctionFrame ? `${blocksPerSecond} blocks/s` : viewNames[newResult.status]} +
+ `; + + this.result = newResult; + return this; + } +} + +class BenchSuiteResultView { + constructor ({runner}) { + this.runner = runner; + this.suite = runner.suite; + this.views = {}; + this.dom = document.createElement('div'); + + const {suite} = runner; + for (const fixture of suite.fixtures) { + this.views[fixture.id] = new BenchResultView({result: new BenchResult({fixture})}); + this.dom.appendChild(this.views[fixture.id].render().dom); + } + + suite.on('result', result => { + this.views[result.fixture.id].update(result); + }); + } + + render () { + return this; + } +} + +window.onload = function () { + const suite = new BenchSuite(); + + suite.add(new BenchFixture({ + projectId: 130041250, + warmUpTime: 0, + recordingTime: 2000 + })); + + suite.add(new BenchFixture({ + projectId: 130041250, + warmUpTime: 4000, + recordingTime: 6000 + })); + + suite.add(new BenchFixture({ + projectId: 14844969, + warmUpTime: 0, + recordingTime: 2000 + })); + + suite.add(new BenchFixture({ + projectId: 14844969, + warmUpTime: 1000, + recordingTime: 6000 + })); + + suite.add(new BenchFixture({ + projectId: 173918262, + warmUpTime: 0, + recordingTime: 5000 + })); + + suite.add(new BenchFixture({ + projectId: 173918262, + warmUpTime: 5000, + recordingTime: 5000 + })); + + suite.add(new BenchFixture({ + projectId: 155128646, + warmUpTime: 0, + recordingTime: 5000 + })); + + suite.add(new BenchFixture({ + projectId: 155128646, + warmUpTime: 5000, + recordingTime: 5000 + })); + + suite.add(new BenchFixture({ + projectId: 89811578, + warmUpTime: 0, + recordingTime: 5000 + })); + + suite.add(new BenchFixture({ + projectId: 89811578, + warmUpTime: 5000, + recordingTime: 5000 + })); + + suite.add(new BenchFixture({ + projectId: 139193539, + warmUpTime: 0, + recordingTime: 5000 + })); + + suite.add(new BenchFixture({ + projectId: 139193539, + warmUpTime: 5000, + recordingTime: 5000 + })); + + const frame = document.getElementsByTagName('iframe')[0]; + const runner = new BenchRunner({frame, suite}); + const resultsView = new BenchSuiteResultView({runner}); + + document.getElementsByClassName('suite-results')[0].appendChild(resultsView.dom); + + runner.run(); +}; From 479f1ba163c1baf56da7cbf5df58a3eee903e23e Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Tue, 16 Jan 2018 18:20:26 -0500 Subject: [PATCH 3/5] Produce a benchmark share link storing the report in the url --- src/playground/benchmark.css | 26 +++++++++++ src/playground/benchmark.js | 89 ++++++++++++++++++++++++++++++++---- src/playground/index.html | 20 ++++++-- 3 files changed, 122 insertions(+), 13 deletions(-) diff --git a/src/playground/benchmark.css b/src/playground/benchmark.css index 07068f59..746c173a 100644 --- a/src/playground/benchmark.css +++ b/src/playground/benchmark.css @@ -11,16 +11,42 @@ p { left: 450px; } } +.share { + display: none; +} +.share label { + cursor: pointer; +} +.share[href] { + display: inline; +} + +.render .profile-tables { + position: static; + left: 0; +} +.render .description { + display: none; +} +.render .run-push { + display: none; +} #scratch-stage { border: 5px solid black; display: block; width: 400px; height: 300px; } +.render #scratch-stage { + display: none; +} .loading label, .profile-count label{ width: 15em; display: inline-block; } +.render .loading { + display: none; +} .profile-tables table { margin: 30px 0 30px 0px; } diff --git a/src/playground/benchmark.js b/src/playground/benchmark.js index bca3e053..3c296397 100644 --- a/src/playground/benchmark.js +++ b/src/playground/benchmark.js @@ -13,12 +13,20 @@ document.querySelector('.run') location.reload(); }, false); +const setShareLink = function (json) { + document.querySelector('.share') + .href = `#view/${btoa(JSON.stringify(json))}`; + document.querySelectorAll('.share')[1] + .href = `suite.html`; +}; + const loadProject = function () { let id = location.hash.substring(1).split(',')[0]; if (id.length < 1 || !isFinite(id)) { id = projectInput.value; } Scratch.vm.downloadProjectId(id); + return id; }; /** @@ -221,6 +229,13 @@ const frameOrder = [ 'Runtime._step' ]; +const trackSlowFrames = [ + 'Sequencer.stepThreads', + 'Sequencer.stepThreads#inner', + 'Sequencer.stepThread', + 'execute' +]; + class FramesTable extends StatTable { constructor (options) { super(options); @@ -241,12 +256,7 @@ class FramesTable extends StatTable { } isSlow (key, frame) { - return ([ - 'Sequencer.stepThreads', - 'Sequencer.stepThreads#inner', - 'Sequencer.stepThread', - 'execute' - ].indexOf(key) > 0 && + return (trackSlowFrames.indexOf(key) > 0 && frame.selfTime / frame.totalTime > SLOW); } } @@ -346,7 +356,7 @@ class ProfilerRun { } run () { - loadProject(); + this.projectId = loadProject(); window.parent.postMessage({ type: 'BENCH_MESSAGE_LOADING' @@ -378,12 +388,49 @@ class ProfilerRun { frames: this.frames.frames, opcodes: this.opcodes.opcodes }, '*'); + + setShareLink({ + fixture: { + projectId: this.projectId, + warmUpTime: this.warmUpTime, + recordingTime: this.maxRecordedTime + }, + frames: this.frames.frames, + opcodes: this.opcodes.opcodes + }); }, 100 + this.warmUpTime + this.maxRecordedTime); }); } + + render (json) { + const {fixture} = json; + document.querySelector('[type=text]').value = [ + fixture.projectId, + fixture.warmUpTime, + fixture.recordingTime + ].join(','); + + this.frames.frames = json.frames.map( + frame => Object.assign(new StatView(), frame, { + name: this.profiler.nameById(this.profiler.idByName(frame.name)) + }) + ); + + this.opcodes.opcodes = {}; + Object.entries(json.opcodes).forEach(([opcode, data]) => { + this.opcodes.opcodes[opcode] = Object.assign(new StatView(), data); + }); + + this.frameTable.render(); + this.opcodeTable.render(); + } } -window.onload = function () { +/** + * Run the benchmark with given parameters in the location's hash field or + * using defaults. + */ +const runBenchmark = function () { // Lots of global variables to make debugging easier // Instantiate the VM. const vm = new window.VirtualMachine(); @@ -493,3 +540,29 @@ window.onload = function () { // Run threads vm.start(); }; + +/** + * Render previously run benchmark data. + * @param {object} json data from a previous benchmark run. + */ +const renderBenchmarkData = function (json) { + const vm = new window.VirtualMachine(); + new ProfilerRun({vm}).render(json); + setShareLink(json); +}; + +window.onload = function () { + if (location.hash.substring(1).startsWith('view')) { + document.body.className = 'render'; + const data = location.hash.substring(6); + const frozen = atob(data); + const json = JSON.parse(frozen); + renderBenchmarkData(json); + } else { + runBenchmark(); + } +}; + +window.onhashchange = function () { + location.reload(); +}; diff --git a/src/playground/index.html b/src/playground/index.html index 81ba4f57..9a0fcfbe 100644 --- a/src/playground/index.html +++ b/src/playground/index.html @@ -8,19 +8,21 @@

Scratch VM Benchmark

-

+

Welcome to the scratch-vm benchmark. This tool helps you profile a scratch project. When you load the page, it: -

    +
    1. loads the default project and enables turbo mode
    2. runs the project for 4 seconds to warm up
    3. profiles for 6 seconds
    4. stops and reports

    - - -

    +

    + + +
    +

    Try a different project, like `130041250`

    @@ -43,6 +45,14 @@ ... + +
    From 775d38d8d5b59f3a35835db22643e4dae6cd603e Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Tue, 16 Jan 2018 18:32:19 -0500 Subject: [PATCH 4/5] Truncate benchmark selfTime and totalTime values --- src/playground/benchmark.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/playground/benchmark.js b/src/playground/benchmark.js index 3c296397..4e6d383b 100644 --- a/src/playground/benchmark.js +++ b/src/playground/benchmark.js @@ -134,10 +134,16 @@ class StatView { } cell = document.createElement('td'); + // Truncate selfTime. Value past the microsecond are floating point + // noise. + this.selfTime = Math.floor(this.selfTime * 1000) / 1000; cell.innerText = (this.selfTime / 1000).toPrecision(3); row.appendChild(cell); cell = document.createElement('td'); + // Truncate totalTime. Value past the microsecond are floating point + // noise. + this.totalTime = Math.floor(this.totalTime * 1000) / 1000; cell.innerText = (this.totalTime / 1000).toPrecision(3); row.appendChild(cell); From 4d4966cba94904fd3be901b107e425c3085b9f69 Mon Sep 17 00:00:00 2001 From: "Michael \"Z\" Goddard" Date: Tue, 16 Jan 2018 18:33:35 -0500 Subject: [PATCH 5/5] Save and compare results, show individual results in iframe --- src/playground/suite.css | 33 +++++ src/playground/suite.html | 6 +- src/playground/suite.js | 267 ++++++++++++++++++++++++++++++++++---- 3 files changed, 278 insertions(+), 28 deletions(-) diff --git a/src/playground/suite.css b/src/playground/suite.css index 08eb5d61..06df9a3f 100644 --- a/src/playground/suite.css +++ b/src/playground/suite.css @@ -5,6 +5,10 @@ html, body { overflow: hidden; } +iframe { + border: none; +} + .runner-controls { position: absolute; width: 30em; @@ -15,6 +19,7 @@ html, body { bottom: 0; overflow: scroll; padding: 1em; + padding-top: 0; box-sizing: border-box; } @@ -28,8 +33,24 @@ html, body { bottom: 0; } +.controls { + margin-bottom: 1em; +} + +.legend { + margin: 1em 0; +} + +.result-view { + border-bottom: 1px solid #ccc; + border-spacing: 0; + border-collapse: collapse; + padding: 5px; +} + .fixture-project { display: inline-block; + clear: both; } .fixture-warm-up { @@ -40,6 +61,18 @@ html, body { display: inline-block; } +.result-view.resume { + cursor: pointer; +} + .result-status { float: right; + text-align: right; + margin-left: 0.3em; +} + +.compare-file { + cursor: pointer; + visibility: hidden; + width: 0; } diff --git a/src/playground/suite.html b/src/playground/suite.html index 8acd7ca6..4100ddd8 100644 --- a/src/playground/suite.html +++ b/src/playground/suite.html @@ -10,11 +10,13 @@

    Scratch VM Benchmark Suite

    - +
    +
    - +
    diff --git a/src/playground/suite.js b/src/playground/suite.js index 11482812..8ab0d492 100644 --- a/src/playground/suite.js +++ b/src/playground/suite.js @@ -70,7 +70,16 @@ class BenchFrameStream extends Emitter { } } +const benchmarkUrlArgs = args => ( + [ + args.projectId, + args.warmUpTime, + args.recordingTime + ].join(',') +); + const BENCH_MESSAGE_TYPE = { + INACTIVE: 'BENCH_MESSAGE_INACTIVE', LOAD: 'BENCH_MESSAGE_LOAD', LOADING: 'BENCH_MESSAGE_LOADING', WARMING_UP: 'BENCH_MESSAGE_WARMING_UP', @@ -84,22 +93,38 @@ class BenchUtil { this.benchStream = new BenchFrameStream(frame); } + setFrameLocation (url) { + this.frame.contentWindow.location.assign(url); + } + startBench (args) { - Promise.resolve() - .then(() => new Promise(resolve => setTimeout(resolve, 100))) + this.benchArgs = args; + this.setFrameLocation(`index.html#${benchmarkUrlArgs(args)}`); + } + + pauseBench () { + new Promise(resolve => setTimeout(resolve, 1000)) .then(() => { - this.frame.contentWindow.location.assign('about:blank'); - }) - .then(() => new Promise(resolve => setTimeout(resolve, 100))) - .then(() => { - this.frame.contentWindow.location.assign( - `/benchmark/#${args.projectId},${args.warmUpTime},${args.recordingTime}`); + this.benchStream.emit('message', { + type: BENCH_MESSAGE_TYPE.INACTIVE + }); }); } + + resumeBench () { + this.startBench(this.benchArgs); + } + + renderResults (results) { + this.setFrameLocation( + `index.html#view/${btoa(JSON.stringify(results))}` + ); + } } const BENCH_STATUS = { INACTIVE: 'BENCH_STATUS_INACTIVE', + RESUME: 'BENCH_STATUS_RESUME', STARTING: 'BENCH_STATUS_STARTING', LOADING: 'BENCH_STATUS_LOADING', WARMING_UP: 'BENCH_STATUS_WARMING_UP', @@ -142,7 +167,9 @@ class BenchFixture extends Emitter { frames: null, opcodes: null }; - if (message.type === BENCH_MESSAGE_TYPE.LOADING) { + if (message.type === BENCH_MESSAGE_TYPE.INACTIVE) { + result.status = BENCH_STATUS.RESUME; + } else if (message.type === BENCH_MESSAGE_TYPE.LOADING) { result.status = BENCH_STATUS.LOADING; } else if (message.type === BENCH_MESSAGE_TYPE.WARMING_UP) { result.status = BENCH_STATUS.WARMING_UP; @@ -232,6 +259,7 @@ class BenchRunner extends Emitter { const viewNames = { [BENCH_STATUS.INACTIVE]: 'Inactive', + [BENCH_STATUS.RESUME]: 'Resume', [BENCH_STATUS.STARTING]: 'Starting', [BENCH_STATUS.LOADING]: 'Loading', [BENCH_STATUS.WARMING_UP]: 'Warming Up', @@ -240,8 +268,10 @@ const viewNames = { }; class BenchResultView { - constructor ({result}) { + constructor ({result, benchUtil}) { this.result = result; + this.compare = null; + this.benchUtil = benchUtil; this.dom = document.createElement('div'); } @@ -249,26 +279,114 @@ class BenchResultView { soon().then(() => this.render(result)); } - render (newResult = this.result) { - const blockFunctionFrame = (newResult.frames ? newResult.frames : []) + resume () { + this.benchUtil.resumeBench(); + } + + setFrameLocation (loc) { + this.benchUtil.pauseBench(); + this.benchUtil.setFrameLocation(loc); + } + + act (ev) { + if ( + ev.type === 'click' && + ev.button === 0 && + !(ev.altKey || ev.ctrlKey || ev.shiftKey || ev.metaKey) + ) { + let target = ev.target; + while (target && target.tagName.toLowerCase() !== 'a') { + target = target.parentElement; + } + if (target && target.tagName.toLowerCase() === 'a') { + if (target.href) { + this.setFrameLocation(target.href); + ev.preventDefault(); + } + } else if (ev.currentTarget.classList.contains('resume')) { + this.resume(); + } + } + } + + render (newResult = this.result, compareResult = this.compare) { + const newResultFrames = (newResult.frames ? newResult.frames : []); + const blockFunctionFrame = newResultFrames .find(frame => frame.name === 'blockFunction'); - const stepThreadsInnerFrame = (newResult.frames ? newResult.frames : []) + const stepThreadsInnerFrame = newResultFrames .find(frame => frame.name === 'Sequencer.stepThreads#inner'); const blocksPerSecond = blockFunctionFrame ? - (blockFunctionFrame.executions / (stepThreadsInnerFrame.totalTime / 1000)) | 0 : + (blockFunctionFrame.executions / + (stepThreadsInnerFrame.totalTime / 1000)) | 0 : + 0; + const stepsPerSecond = stepThreadsInnerFrame ? + (stepThreadsInnerFrame.executions / + (stepThreadsInnerFrame.totalTime / 1000)) | 0 : 0; + const compareResultFrames = ( + compareResult && compareResult.frames ? + compareResult.frames : + [] + ); + const blockFunctionCompareFrame = compareResultFrames + .find(frame => frame.name === 'blockFunction'); + const stepThreadsInnerCompareFrame = compareResultFrames + .find(frame => frame.name === 'Sequencer.stepThreads#inner'); + + const compareBlocksPerSecond = blockFunctionCompareFrame ? + (blockFunctionCompareFrame.executions / + (stepThreadsInnerCompareFrame.totalTime / 1000)) | 0 : + 0; + const compareStepsPerSecond = stepThreadsInnerCompareFrame ? + (stepThreadsInnerCompareFrame.executions / + (stepThreadsInnerCompareFrame.totalTime / 1000)) | 0 : + 0; + + const statusName = viewNames[newResult.status]; + + this.dom.className = `result-view ${ + viewNames[newResult.status].toLowerCase() + }`; + this.dom.onclick = this.act.bind(this); + let url = `index.html#${benchmarkUrlArgs(newResult.fixture)}`; + if (newResult.status === BENCH_STATUS.COMPLETE) { + url = `index.html#view/${btoa(JSON.stringify(newResult))}`; + } + let compareUrl = url; + if (compareResult && compareResult) { + compareUrl = + `index.html#view/${btoa(JSON.stringify(compareResult))}`; + } + let compareHTML = ''; + if (stepThreadsInnerFrame && stepThreadsInnerCompareFrame) { + compareHTML = ` +
    +
    ${compareStepsPerSecond}
    +
    ${compareBlocksPerSecond}
    +
    +     `; + } this.dom.innerHTML = `
    - ${newResult.fixture.projectId}
    - (${newResult.fixture.warmUpTime / 1000}s - ${newResult.fixture.recordingTime / 1000}s)
    - ${blockFunctionFrame ? `${blocksPerSecond} blocks/s` : viewNames[newResult.status]} +
    ${stepThreadsInnerFrame ? `steps/s` : ''}
    +
    ${blockFunctionFrame ? `blocks/s` : statusName}
    +
    + +
    +
    ${stepThreadsInnerFrame ? `${stepsPerSecond}` : ''}
    +
    ${blockFunctionFrame ? `${blocksPerSecond}` : ''}
    +
    +     + ${compareHTML} +
    + Run for ${newResult.fixture.recordingTime / 1000} seconds after + ${newResult.fixture.warmUpTime / 1000} seconds
    `; @@ -279,15 +397,18 @@ class BenchResultView { class BenchSuiteResultView { constructor ({runner}) { + const {suite, util} = runner; + this.runner = runner; - this.suite = runner.suite; + this.suite = suite; this.views = {}; this.dom = document.createElement('div'); - const {suite} = runner; for (const fixture of suite.fixtures) { - this.views[fixture.id] = new BenchResultView({result: new BenchResult({fixture})}); - this.dom.appendChild(this.views[fixture.id].render().dom); + this.views[fixture.id] = new BenchResultView({ + result: new BenchResult({fixture}), + benchUtil: util + }); } suite.on('result', result => { @@ -296,12 +417,93 @@ class BenchSuiteResultView { } render () { + this.dom.innerHTML = `
    + Project ID +
    +
    steps per second
    +
    blocks per second
    +
    +
    Description
    +
    + +
    +   +
    + +
    +
    + +
    +
    `; + + for (const fixture of this.suite.fixtures) { + this.dom.appendChild(this.views[fixture.id].render().dom); + } + return this; } } +let suite; +let suiteView; + +window.upload = function (_this) { + if (!_this.files.length) { + return; + } + const reader = new FileReader(); + reader.onload = function () { + const report = JSON.parse(reader.result); + Object.values(suiteView.views) + .forEach(view => { + const sameFixture = report.results.find(result => ( + result.fixture.projectId === + view.result.fixture.projectId && + result.fixture.warmUpTime === + view.result.fixture.warmUpTime && + result.fixture.recordingTime === + view.result.fixture.recordingTime + )); + + if (sameFixture) { + if ( + view.result && view.result.frames && + view.result.frames.length > 0 + ) { + view.render(view.result, sameFixture); + } else { + view.compare = sameFixture; + } + } + }); + }; + reader.readAsText(_this.files[0]); +}; + +window.download = function (_this) { + const blob = new Blob([JSON.stringify({ + meta: { + source: 'Scratch VM Benchmark Suite', + version: 1 + }, + results: Object.values(suiteView.views) + .map(view => view.result) + .filter(view => view.status === BENCH_STATUS.COMPLETE) + })], {type: 'application/json'}); + + _this.download = 'scratch-vm-benchmark.json'; + _this.href = URL.createObjectURL(blob); +}; + window.onload = function () { - const suite = new BenchSuite(); + suite = new BenchSuite(); suite.add(new BenchFixture({ projectId: 130041250, @@ -375,11 +577,24 @@ window.onload = function () { recordingTime: 5000 })); + suite.add(new BenchFixture({ + projectId: 187694931, + warmUpTime: 0, + recordingTime: 5000 + })); + + suite.add(new BenchFixture({ + projectId: 187694931, + warmUpTime: 5000, + recordingTime: 5000 + })); + const frame = document.getElementsByTagName('iframe')[0]; const runner = new BenchRunner({frame, suite}); - const resultsView = new BenchSuiteResultView({runner}); + const resultsView = suiteView = new BenchSuiteResultView({runner}).render(); - document.getElementsByClassName('suite-results')[0].appendChild(resultsView.dom); + document.getElementsByClassName('suite-results')[0] + .appendChild(resultsView.dom); runner.run(); };