ScCN/packager
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace project downloader with SBDL

Browse Source
This commit is contained in:
GarboMuffin committed 2022-08-14 18:03:57 -05:00
1 parent 3efd8f97f9
commit 56646e4fdf
5 files changed
+53 -367
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

package-lock.json
Generated
+31 -3
View File
@@ -1791,6 +1791,34 @@
"integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==",
"dev": true
},
"@turbowarp/json": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/@turbowarp/json/-/json-0.1.1.tgz",
"integrity": "sha512-6APLbD+TGNdVKUu2UlWM1cgHbwrStWIwpvDVdiqS/SJ/i0OfjtqeAhnK5EjqNazXGWI+p68ZNxMJe2+UtZZSzA=="
},
"@turbowarp/sbdl": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/@turbowarp/sbdl/-/sbdl-2.3.0.tgz",
"integrity": "sha512-N7p9LjeFTnD8qQocwAxHn5MIa/9b8yxiznLwcc1TLbBEWyJoTEZY63L1Y567bLj+8HttH7sohSizZcUvQ0nPGw==",
"requires": {
"@turbowarp/json": "^0.1.1",
"cross-fetch": "^3.1.5",
"jszip": "^3.10.1"
},
"dependencies": {
"jszip": {
"version": "3.10.1",
"resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz",
"integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==",
"requires": {
"lie": "~3.3.0",
"pako": "~1.0.2",
"readable-stream": "~2.3.6",
"setimmediate": "^1.0.5"
}
}
}
},
"@types/babel__core": {
"version": "7.1.19",
"resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.19.tgz",
@@ -9662,7 +9690,7 @@
},
"scratch-render": {
"version": "git+https://github.com/TurboWarp/scratch-render.git#8d234dad9d9093973b445057791ae47afec3c1b1",
"from": "git+https://github.com/TurboWarp/scratch-render.git#8d234dad9d9093973b445057791ae47afec3c1b1",
"from": "git+https://github.com/TurboWarp/scratch-render.git#develop",
"dev": true,
"requires": {
"grapheme-breaker": "0.3.2",
@@ -9764,8 +9792,8 @@
"dev": true
},
"scratch-vm": {
"version": "git+https://github.com/TurboWarp/scratch-vm.git#6db9599f2e7788bd2599710f993c29bb30c737f7",
"from": "git+https://github.com/TurboWarp/scratch-vm.git#6db9599f2e7788bd2599710f993c29bb30c737f7",
"version": "git+https://github.com/TurboWarp/scratch-vm.git#dc9b3f60558de9626761ae698083b14ee9a8ef2c",
"from": "git+https://github.com/TurboWarp/scratch-vm.git#develop",
"dev": true,
"requires": {
"@vernier/godirect": "1.5.0",
package.json
+1
View File
@@ -33,6 +33,7 @@
"license": "Apache-2.0",
"dependencies": {
"@fiahfy/icns": "0.0.7",
"@turbowarp/sbdl": "^2.3.0",
"cross-fetch": "^3.1.4",
"jszip": "^3.7.1",
"sha.js": "^2.4.11"
src/packager/download-project.js
+19 -300
View File
@@ -1,43 +1,5 @@
import JSZip from 'jszip';
import {EventTarget, CustomEvent} from '../common/event-target';
import optimizeSb3Json from './minify/sb3';
import fetchAsArrayBuffer from './safer-fetch';
const ASSET_HOST = 'https://assets.scratch.mit.edu/internalapi/asset/$path/get/';
// Browser support for Array.prototype.flat is not good enough yet
const flat = (array) => {
const result = [];
for (const i of array) {
if (Array.isArray(i)) {
for (const j of i) {
result.push(j);
}
} else {
result.push(i);
}
}
return result;
};
const identifyProjectType = (projectData) => {
if ('targets' in projectData) {
return 'sb3';
} else if ('objName' in projectData) {
return 'sb2';
}
return null;
};
const isScratch1Project = (uint8array) => {
const MAGIC = 'ScratchV';
for (let i = 0; i < MAGIC.length; i++) {
if (uint8array[i] !== MAGIC.charCodeAt(i)) {
return false;
}
}
return true;
};
import {downloadProjectFromBuffer} from '@turbowarp/sbdl';
const unknownAnalysis = () => ({
stageVariables: [],
@@ -100,275 +62,32 @@ const mutateScratch3InPlace = (projectData) => {
makeImpliedCloudVariables(projectData);
optimizeSb3Json(projectData);
return projectData;
};
const loadScratch2 = (projectData, progressTarget) => {
const IMAGE_EXTENSIONS = ['svg', 'png', 'jpg', 'jpeg', 'bmp'];
const SOUND_EXTENSIONS = ['wav', 'mp3'];
export const downloadProject = async (projectData, progressCallback) => {
let analysis = unknownAnalysis();
const zip = new JSZip();
const options = {
onProgress(type, loaded, total) {
progressCallback(type, loaded, total);
},
// sb2 files have two ways of storing references to files.
// In the online editor they use md5 hashes ("md5ext" because they also have an extension).
// In the offline editor they use separate integer file IDs for images and sounds.
// We need the sb2 to use those integer file IDs, but the ones from the Scratch API don't have those, so we create them ourselves
let soundAccumulator = 0;
let imageAccumulator = 0;
const getExtension = (md5ext) => md5ext.split('.')[1] || '';
const nextId = (md5) => {
const extension = getExtension(md5);
if (IMAGE_EXTENSIONS.includes(extension)) {
return imageAccumulator++;
} else if (SOUND_EXTENSIONS.includes(extension)) {
return soundAccumulator++;
}
console.warn('unknown extension: ' + extension);
return imageAccumulator++;
};
const fetchAndStoreAsset = (md5ext, id) => {
progressTarget.dispatchEvent(new CustomEvent('asset-fetch', {
detail: md5ext
}));
return fetchAsArrayBuffer(ASSET_HOST.replace('$path', md5ext))
.then((arrayBuffer) => {
const path = `${id}.${getExtension(md5ext)}`;
zip.file(path, arrayBuffer);
progressTarget.dispatchEvent(new CustomEvent('asset-fetched', {
detail: md5ext
}));
});
};
const downloadAssets = (assets) => {
const md5extToId = new Map();
const handleAsset = (md5ext) => {
if (!md5extToId.has(md5ext)) {
md5extToId.set(md5ext, nextId(md5ext));
}
return md5extToId.get(md5ext);
};
for (const asset of assets) {
if (asset.md5) {
asset.soundID = handleAsset(asset.md5);
}
if (asset.baseLayerMD5) {
asset.baseLayerID = handleAsset(asset.baseLayerMD5);
}
if (asset.textLayerMD5) {
asset.textLayerID = handleAsset(asset.textLayerMD5);
}
}
return Promise.all(Array.from(md5extToId.entries()).map(([md5ext, id]) => fetchAndStoreAsset(md5ext, id)));
};
const targets = [
projectData,
...projectData.children.filter((c) => !c.listName && !c.target)
];
const costumes = flat(targets.map((i) => i.costumes || []));
const sounds = flat(targets.map((i) => i.sounds || []));
return downloadAssets([...costumes, ...sounds])
.then(() => {
// Project JSON is mutated during loading, so add it at the e nd.
zip.file('project.json', JSON.stringify(projectData));
return {
zip,
analysis: analyzeScratch2(projectData)
};
});
};
const loadScratch3 = async (projectData, progressTarget) => {
/**
* @typedef SB3Asset Raw costume or sound data from an sb3 project.json.
* @property {string} assetId md5 checksum of the asset (eg. b7b7898cfcd9ba13e89a4e74dd56a1ff)
* @property {string} dataFormat file extension of the asset (eg. svg, wav)
* @property {string|undefined} md5ext dataFormat (eg. b7b7898cfcd9ba13e89a4e74dd56a1ff.svg)
* md5ext is not guaranteed to exist.
* There are additional properties that we don't care about.
*/
const zip = new JSZip();
/**
* @param {SB3Asset[]} assets
* @returns {SB3Asset[]}
*/
const prepareAssets = (assets) => {
const result = [];
const knownIds = new Set();
for (const data of assets) {
// Make sure md5ext always exists.
// See the "Cake" costume of https://projects.scratch.mit.edu/630358355 for an example.
// https://github.com/forkphorus/forkphorus/issues/504
if (!data.md5ext) {
data.md5ext = `${data.assetId}.${data.dataFormat}`;
}
// Deduplicate assets so we don't make unnecessary requests later.
// Use md5ext instead of assetId because there are a few projects that have assets with the same
// assetId but different md5ext. (eg. https://scratch.mit.edu/projects/531881458)
const md5ext = data.md5ext;
if (knownIds.has(md5ext)) {
continue;
}
knownIds.add(md5ext);
result.push(data);
}
return result;
};
/**
* @param {SB3Asset} data
* @returns {Promise<void>}
*/
const addFile = async (data) => {
// prepareAssets will guarantee md5ext exists
const md5ext = data.md5ext;
progressTarget.dispatchEvent(new CustomEvent('asset-fetch', {
detail: md5ext
}));
const buffer = await fetchAsArrayBuffer(ASSET_HOST.replace('$path', md5ext));
zip.file(md5ext, buffer);
progressTarget.dispatchEvent(new CustomEvent('asset-fetched', {
detail: md5ext
}));
};
zip.file('project.json', JSON.stringify(mutateScratch3InPlace(projectData)));
const targets = projectData.targets;
const costumes = flat(targets.map((t) => t.costumes || []));
const sounds = flat(targets.map((t) => t.sounds || []));
const assets = prepareAssets([...costumes, ...sounds]);
await Promise.all(assets.map(addFile));
return {
zip,
analysis: analyzeScratch3(projectData)
};
};
const downloadJSONProject = (json, progressTarget) => {
const type = identifyProjectType(json);
if (type === 'sb3') {
return loadScratch3(json, progressTarget);
} else if (type === 'sb2') {
return loadScratch2(json, progressTarget);
} else {
throw new Error('Unknown project type');
}
};
export const downloadProject = async (data, progressCallback = () => {}) => {
let type;
let arrayBuffer;
let analysis;
const bufferView = new Uint8Array(data);
if (bufferView[0] === '{'.charCodeAt(0)) {
// JSON project, we have to download all the assets
const progressTarget = new EventTarget();
let isDoneLoadingProject = false;
let timeout = null;
let loadedAssets = 0;
let totalAssets = 0;
const sendThrottledAssetProgressUpdate = () => {
if (timeout) {
return;
}
timeout = setTimeout(() => {
timeout = null;
if (!isDoneLoadingProject) {
progressCallback('assets', loadedAssets, totalAssets);
}
});
};
progressTarget.addEventListener('asset-fetch', () => {
totalAssets++;
sendThrottledAssetProgressUpdate();
});
progressTarget.addEventListener('asset-fetched', () => {
loadedAssets++;
sendThrottledAssetProgressUpdate();
});
const text = new TextDecoder().decode(data);
const json = JSON.parse(text);
type = identifyProjectType(json);
const downloaded = await downloadJSONProject(json, progressTarget);
progressCallback('assets', totalAssets, totalAssets);
isDoneLoadingProject = true;
arrayBuffer = await downloaded.zip.generateAsync({
type: 'arraybuffer',
compression: 'DEFLATE'
}, (meta) => {
progressCallback('compress', meta.percent / 100);
});
analysis = downloaded.analysis;
} else {
if (isScratch1Project(bufferView)) {
arrayBuffer = data;
analysis = unknownAnalysis();
} else {
let zip;
try {
zip = await JSZip.loadAsync(data);
} catch (e) {
throw new Error('Cannot parse project: not a zip or sb');
}
const projectDataFile = zip.file(/^([^/]*\/)?project\.json$/)[0];
if (!projectDataFile) {
throw new Error('project.json is missing');
}
const pathPrefix = projectDataFile.name.substr(0, projectDataFile.name.indexOf('project.json'));
if (pathPrefix !== '') {
zip = zip.folder(pathPrefix);
}
const projectDataText = await projectDataFile.async('text');
const projectData = JSON.parse(projectDataText);
type = identifyProjectType(projectData);
processJSON(type, projectData) {
if (type === 'sb3') {
zip.file('project.json', JSON.stringify(mutateScratch3InPlace(projectData)));
arrayBuffer = await zip.generateAsync({
type: 'arraybuffer',
compression: 'DEFLATE'
}, (meta) => {
progressCallback('compress', meta.percent / 100);
});
mutateScratch3InPlace(projectData);
analysis = analyzeScratch3(projectData);
} else {
arrayBuffer = data;
return projectData;
}
if (type === 'sb2') {
analysis = analyzeScratch2(projectData);
}
}
}
if (type === 'sb3') {
return {
type: 'sb3',
arrayBuffer,
analysis
}
}
return {
type: 'blob',
arrayBuffer,
analysis
};
const project = await downloadProjectFromBuffer(projectData, options);
if (project.type !== 'sb3') {
project.type = 'blob';
}
project.analysis = analysis;
return project;
};
src/packager/safer-fetch.js
-63
View File
@@ -1,63 +0,0 @@
import fetch from 'cross-fetch';
// Based on https://github.com/TurboWarp/scratch-storage/blob/develop/src/safer-fetch.js
// This throttles and retries fetch() to mitigate the effect of random network errors and
// random browser errors (especially in Chrome)
let currentFetches = 0;
const queue = [];
const MAX_ATTEMPTS = 3;
const MAX_CONCURRENT = 100;
const RETRY_DELAY = 5000;
const startNextFetch = ([resolve, url, options]) => {
let firstError;
let attempts = 0;
const attemptToFetch = () => fetch(url, options)
.then((r) => {
if (r.ok) {
return r.arrayBuffer()
}
throw new Error(`Unexpected status code: ${r.status}`);
})
.then((buffer) => {
currentFetches--;
checkStartNextFetch();
return buffer;
})
.catch((error) => {
console.warn(`Attempt to fetch ${url} failed`, error);
if (!firstError) {
firstError = error;
}
if (attempts < MAX_ATTEMPTS) {
attempts++;
return new Promise((cb) => setTimeout(cb, (attempts + Math.random() - 1) * RETRY_DELAY))
.then(attemptToFetch);
}
currentFetches--;
checkStartNextFetch();
throw new Error(`Failed to fetch ${url}: ${firstError}`);
});
return resolve(attemptToFetch());
};
const checkStartNextFetch = () => {
if (currentFetches < MAX_CONCURRENT && queue.length > 0) {
currentFetches++;
startNextFetch(queue.shift());
}
};
const saferFetchAsArrayBuffer = (url, options) => new Promise((resolve) => {
queue.push([resolve, url, options]);
checkStartNextFetch();
});
export default saferFetchAsArrayBuffer;
webpack.config.js
+2 -1
View File
@@ -225,7 +225,8 @@ const makeNode = () => ({
jszip: 'jszip',
'cross-fetch': 'cross-fetch',
'sha.js': 'sha.js',
'@fiahfy/icns': '@fiahfy/icns'
'@fiahfy/icns': '@fiahfy/icns',
'@turbowarp/sbdl': '@turbowarp/sbdl'
},
module: {
rules: [