Lightly optimize sb3 project.json

This uses some disabled-by-default code from TW/scratch-vm to optimize projects
It's not optimal, but it's safe outside of the editor while still saving significant space
Part of #148

src/packager/download-project.js

@@ -1,5 +1,6 @@
 import JSZip from 'jszip';
 import EventTarget from '../common/event-target';
+import optimizeSb3Json from './minify/sb3';
 
 const ASSET_HOST = 'https://assets.scratch.mit.edu/internalapi/asset/$path/get/';
 
@@ -205,7 +206,7 @@ const loadScratch3 = (projectData, progressTarget) => {
     return result;
   }
 
-  zip.file('project.json', JSON.stringify(projectData));
+  zip.file('project.json', JSON.stringify(optimizeSb3Json(projectData)));
 
   const targets = projectData.targets;
   const costumes = [].concat.apply([], targets.map((t) => t.costumes || []));
@@ -299,6 +300,7 @@ export const downloadProject = async (data, progressCallback) => {
       const projectData = JSON.parse(projectDataText);
       type = identifyProjectType(projectData);
       if (type === 'sb3') {
+        zip.file('project.json', JSON.stringify(optimizeSb3Json(projectData)));
         analysis = analyzeScratch3(projectData);
       } else {
         analysis = analyzeScratch2(projectData);

src/packager/minify/sb3.js

@@ -0,0 +1,211 @@
+// Currently, the optimization here is intentionally not optimal
+// We had some corruption problems in the past that we are trying to avoid
+
+const SOUP = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!#%()*+,-./:;=?@[]^_`{|}~';
+const generateId = i => {
+  // IDs in Object.keys(vm.runtime.monitorBlocks._blocks) already have meaning, so make sure to skip those
+  // We don't bother listing many here because most would take more than ten million items to be used
+  if (i > 1309) i++; // of
+  let str = '';
+  while (i >= 0) {
+    str = SOUP[i % SOUP.length] + str;
+    i = Math.floor(i / SOUP.length) - 1;
+  }
+  return str;
+};
+
+class Pool {
+  constructor() {
+    this.generatedIds = new Map();
+    this.references = new Map();
+  }
+  addReference(id) {
+    const currentCount = this.references.get(id) || 0;
+    this.references.set(id, currentCount + 1);
+  }
+  generateNewIds() {
+    const entries = Array.from(this.references.entries());
+    // The most used original IDs should get the shortest new IDs.
+    entries.sort((a, b) => b[1] - a[1]);
+    for (let i = 0; i < entries.length; i++) {
+      this.generatedIds.set(entries[i][0], generateId(i));
+    }
+  }
+  getNewId(originalId) {
+    if (this.generatedIds.has(originalId)) {
+      return this.generatedIds.get(originalId);
+    }
+    return originalId;
+  }
+}
+
+const optimizeSb3Json = (projectData) => {
+  const pool = new Pool();
+
+  for (const monitor of projectData.monitors) {
+    const monitorOpcode = monitor.opcode;
+    if (monitorOpcode === 'data_variable' || monitorOpcode === 'data_listcontents') {
+      const monitorId = monitor.id;
+      pool.addReference(monitorId);
+    }
+  }
+  for (const target of projectData.targets) {
+    const handleCompressedNative = native => {
+      const type = native[0];
+      if (type === /* VAR_PRIMITIVE */ 12 || type === /* LIST_PRIMITIVE */ 13) {
+        const variableId = native[2];
+        pool.addReference(variableId);
+      } else if (type === /* BROADCAST_PRIMITIVE */ 11) {
+        const broadcastId = native[2];
+        pool.addReference(broadcastId);
+      }
+    };
+
+    for (const variableId of Object.keys(target.variables)) {
+      pool.addReference(variableId);
+    }
+    for (const variableId of Object.keys(target.lists)) {
+      pool.addReference(variableId);
+    }
+    for (const broadcastId of Object.keys(target.broadcasts)) {
+      pool.addReference(broadcastId);
+    }
+    for (const blockId of Object.keys(target.blocks)) {
+      const block = target.blocks[blockId];
+      pool.addReference(blockId);
+      if (Array.isArray(block)) {
+        handleCompressedNative(block);
+        continue;
+      }
+      if (block.parent) {
+        pool.addReference(block.parent);
+      }
+      if (block.next) {
+        pool.addReference(block.next);
+      }
+      if (block.comment) {
+        pool.addReference(block.comment);
+      }
+      if (block.fields.VARIABLE) {
+        pool.addReference(block.fields.VARIABLE[1]);
+      }
+      if (block.fields.LIST) {
+        pool.addReference(block.fields.LIST[1]);
+      }
+      if (block.fields.BROADCAST_OPTION) {
+        pool.addReference(block.fields.BROADCAST_OPTION[1]);
+      }
+      for (const inputName of Object.keys(block.inputs)) {
+        const input = block.inputs[inputName];
+        const inputValue = input[1];
+        if (Array.isArray(inputValue)) {
+          handleCompressedNative(inputValue);
+        } else if (typeof inputValue === 'string') {
+          const childBlockId = input[1];
+          pool.addReference(childBlockId);
+        }
+      }
+    }
+    for (const commentId of Object.keys(target.comments)) {
+      const comment = target.comments[commentId];
+      pool.addReference(commentId);
+      if (comment.blockId) {
+        pool.addReference(comment.blockId);
+      }
+    }
+  }
+
+  // Used the data from the first scan to replace all the IDs with shorter versions
+  pool.generateNewIds();
+  for (const monitor of projectData.monitors) {
+    const monitorOpcode = monitor.opcode;
+    if (monitorOpcode === 'data_variable' || monitorOpcode === 'data_listcontents') {
+      const monitorId = monitor.id;
+      monitor.id = pool.getNewId(monitorId);
+    }
+  }
+  for (const target of projectData.targets) {
+    const newVariables = {};
+    const newLists = {};
+    const newBroadcasts = {};
+    const newBlocks = {};
+    const newComments = {};
+
+    const handleCompressedNative = native => {
+      const type = native[0];
+      if (type === /* VAR_PRIMITIVE */ 12 || type === /* LIST_PRIMITIVE */ 13) {
+        const variableId = native[2];
+        native[2] = pool.getNewId(variableId);
+      } else if (type === /* BROADCAST_PRIMITIVE */ 11) {
+        const broadcastId = native[2];
+        native[2] = pool.getNewId(broadcastId);
+      }
+    };
+
+    for (const variableId of Object.keys(target.variables)) {
+      const variable = target.variables[variableId];
+      newVariables[pool.getNewId(variableId)] = variable;
+    }
+    for (const variableId of Object.keys(target.lists)) {
+      const variable = target.lists[variableId];
+      newLists[pool.getNewId(variableId)] = variable;
+    }
+    for (const broadcastId of Object.keys(target.broadcasts)) {
+      const broadcast = target.broadcasts[broadcastId];
+      newBroadcasts[pool.getNewId(broadcastId)] = broadcast;
+    }
+    for (const blockId of Object.keys(target.blocks)) {
+      const block = target.blocks[blockId];
+      newBlocks[pool.getNewId(blockId)] = block;
+      if (Array.isArray(block)) {
+        handleCompressedNative(block);
+        continue;
+      }
+      if (block.parent) {
+        block.parent = pool.getNewId(block.parent);
+      }
+      if (block.next) {
+        block.next = pool.getNewId(block.next);
+      }
+      if (block.comment) {
+        block.comment = pool.getNewId(block.comment);
+      }
+      if (block.fields.VARIABLE) {
+        block.fields.VARIABLE[1] = pool.getNewId(block.fields.VARIABLE[1]);
+      }
+      if (block.fields.LIST) {
+        block.fields.LIST[1] = pool.getNewId(block.fields.LIST[1]);
+      }
+      if (block.fields.BROADCAST_OPTION) {
+        block.fields.BROADCAST_OPTION[1] = pool.getNewId(block.fields.BROADCAST_OPTION[1]);
+      }
+      for (const inputName of Object.keys(block.inputs)) {
+        const input = block.inputs[inputName];
+        const inputValue = input[1];
+        if (Array.isArray(inputValue)) {
+          handleCompressedNative(inputValue);
+        } else if (typeof inputValue === 'string') {
+          const childBlockId = input[1];
+          input[1] = pool.getNewId(childBlockId);
+        }
+      }
+    }
+    for (const commentId of Object.keys(target.comments)) {
+      const comment = target.comments[commentId];
+      newComments[pool.getNewId(commentId)] = comment;
+      if (comment.blockId) {
+        comment.blockId = pool.getNewId(comment.blockId);
+      }
+    }
+
+    target.variables = newVariables;
+    target.lists = newLists;
+    target.broadcasts = newBroadcasts;
+    target.blocks = newBlocks;
+    target.comments = newComments;
+  }
+
+  return projectData;
+};
+
+export default optimizeSb3Json;