New approach to handling addon libraries-raw and assets

They are now handled with `import` just like everything else in Scratch.
Benefits of this:
 - Addon assets can be cached immutably now
 - No special webpack configuration needed anymore

src/addons/README.md

@@ -14,7 +14,6 @@ Directory structure:
 
  - addons - the addons (managed by pull.js)
  - addons-l10n - addon translations (managed by pull.js)
- - libraries - libraries used by addons loaded with `import` (managed by pull.js)
- - libraries-raw - libraries used by addons loaded with `addon.tab.loadScript` (managed by pull.js)
+ - libraries - libraries used by addons (managed by pull.js)
  - api-libraries - libraries used by addon API
  - settings - the settings page and its translations

src/addons/addons/clones/userscript.js

@@ -3,6 +3,15 @@
  * @license GPLv3.0 (see LICENSE_GPL or https://www.gnu.org/licenses/ for more information)
  */
 
+/* inserted by pull.js */
+import _twAsset0 from "./300cats.svg";
+import _twAsset1 from "./cat.svg";
+const _twGetAsset = (path) => {
+  if (path === "/300cats.svg") return _twAsset0;
+  if (path === "/cat.svg") return _twAsset1;
+  throw new Error(`Unknown asset: ${path}`);
+};
+
 export default async function ({ addon, global, console, msg }) {
   console.log("clones counter enabled");
 
@@ -44,10 +53,10 @@ export default async function ({ addon, global, console, msg }) {
 
     if (v === 300) {
       count.style.color = "#ff6680";
-      icon.src = addon.self.dir + "/300cats.svg";
+      icon.src = /* changed by pull.js */ _twGetAsset("/300cats.svg");
     } else {
       count.style.color = "";
-      icon.src = addon.self.dir + "/cat.svg";
+      icon.src = /* changed by pull.js */ _twGetAsset("/cat.svg");
     }
     count.innerText = msg("clones", { cloneCount: v });
   }
@@ -58,7 +67,7 @@ export default async function ({ addon, global, console, msg }) {
     if (addon.tab.editorMode === "editor") {
       doCloneChecks(vm.runtime._cloneCounter);
 
-      icon.src = addon.self.dir + "/cat.svg";
+      icon.src = /* changed by pull.js */ _twGetAsset("/cat.svg");
 
       bar.appendChild(countContainerContainer);
     }

src/addons/addons/color-picker/userscript.js

@@ -3,6 +3,13 @@
  * @license GPLv3.0 (see LICENSE_GPL or https://www.gnu.org/licenses/ for more information)
  */
 
+/* inserted by pull.js */
+import _twScript0 from "!file-loader!../../libraries/tinycolor-min.js";
+const _twGetAsset = (path) => {
+  if (path === "/tinycolor-min.js") return _twScript0;
+  throw new Error(`Unknown asset: ${path}`);
+};
+
 import codeEditorHandler from "./code-editor.js";
 import paintEditorHandler from "./paint-editor.js";
 
@@ -10,7 +17,7 @@ import paintEditorHandler from "./paint-editor.js";
 // Note that we don't await other scripts (they block!)
 export default async (api) => {
   const { addon } = api;
-  await addon.tab.loadScript(addon.self.lib + "/tinycolor-min.js");
+  await addon.tab.loadScript(/* changed by pull.js */ _twGetAsset("/tinycolor-min.js"));
   codeEditorHandler(api);
   paintEditorHandler(api);
 };

src/addons/addons/hide-flyout/userscript.js

@@ -3,6 +3,15 @@
  * @license GPLv3.0 (see LICENSE_GPL or https://www.gnu.org/licenses/ for more information)
  */
 
+/* inserted by pull.js */
+import _twAsset0 from "./lock.svg";
+import _twAsset1 from "./unlock.svg";
+const _twGetAsset = (path) => {
+  if (path === "/lock.svg") return _twAsset0;
+  if (path === "/unlock.svg") return _twAsset1;
+  throw new Error(`Unknown asset: ${path}`);
+};
+
 export default async function ({ addon, global, console }) {
   var placeHolderDiv = null,
     lockDisplay = null,
@@ -21,11 +30,11 @@ export default async function ({ addon, global, console }) {
       // Lock Img
       if (lockDisplay) lockDisplay.remove();
       lockDisplay = document.createElement("img");
-      lockDisplay.src = addon.self.dir + "/unlock.svg";
+      lockDisplay.src = /* changed by pull.js */ _twGetAsset("/unlock.svg");
       lockDisplay.className = "sa-lock-image";
       lockDisplay.onclick = () => {
         flyoutLock = !flyoutLock;
-        lockDisplay.src = addon.self.dir + `/${flyoutLock ? "" : "un"}lock.svg`;
+        lockDisplay.src = /* changed by pull.js */ _twGetAsset(`/${flyoutLock ? "" : "un"}lock.svg`);
       };
 
       function positionElements() {

src/addons/addons/onion-skinning/userscript.js

@@ -3,6 +3,19 @@
  * @license GPLv3.0 (see LICENSE_GPL or https://www.gnu.org/licenses/ for more information)
  */
 
+/* inserted by pull.js */
+import _twAsset0 from "./decrement.svg";
+import _twAsset1 from "./increment.svg";
+import _twAsset2 from "./settings.svg";
+import _twAsset3 from "./toggle.svg";
+const _twGetAsset = (path) => {
+  if (path === "/decrement.svg") return _twAsset0;
+  if (path === "/increment.svg") return _twAsset1;
+  if (path === "/settings.svg") return _twAsset2;
+  if (path === "/toggle.svg") return _twAsset3;
+  throw new Error(`Unknown asset: ${path}`);
+};
+
 export default async function ({ addon, global, console, msg }) {
   let project = null;
   let paperCanvas = null;
@@ -539,7 +552,7 @@ export default async function ({ addon, global, console, msg }) {
     el.className = "sa-onion-image";
     el.draggable = false;
     el.dataset.image = name;
-    el.src = addon.self.dir + "/" + name + ".svg";
+    el.src = /* changed by pull.js */ _twGetAsset("/" + name + ".svg");
     return el;
   };
 

src/addons/api.js

@@ -165,17 +165,12 @@ class Settings extends EventTarget {
 }
 
 class Self {
-    constructor (id) {
-        this.dir = `addon-files/${id}`;
-        this.lib = 'addon-files/libraries-raw';
+    // These are removed at build-time by pull.js. Throw if attempting to access them at runtime.
+    get dir () {
+        throw new Error(`Addon tried to access addon.self.dir`);
     }
-}
-
-class Addon {
-    constructor (addonId, manifest) {
-        this.tab = new Tab();
-        this.settings = new Settings(addonId, manifest);
-        this.self = new Self(addonId);
+    get lib () {
+        throw new Error(`Addon tried to access addon.self.lib`);
     }
 }
 
@@ -190,7 +185,11 @@ class AddonRunner {
         this.publicAPI = {
             global,
             console,
-            addon: new Addon(id, manifest),
+            addon: {
+                tab: new Tab(),
+                settings: new Settings(id, manifest),
+                self: new Self()
+            },
             msg: this.msg.bind(this),
             safeMsg: this.safeMsg.bind(this)
         };
@@ -238,12 +237,17 @@ class AddonRunner {
         }
     }
 
-    _run () {
+    async _run () {
         this.updateCSSVariables();
 
         if (this.manifest.userstyles) {
             for (const userstyle of this.manifest.userstyles) {
-                const source = require(`./addons/${this.id}/${userstyle.url}`);
+                const m = await import(
+                    /* webpackInclude: /\.css$/ */
+                    /* webpackMode: "eager" */
+                    `!raw-loader!./addons/${this.id}/${userstyle.url}`
+                );
+                const source = m.default;
                 const style = createStylesheet(source);
                 style.className = 'scratch-addons-theme';
                 style.dataset.addonId = this.id;
@@ -254,17 +258,18 @@ class AddonRunner {
 
         if (this.manifest.userscripts) {
             for (const userscript of this.manifest.userscripts) {
-                require(`./addons/${this.id}/${userscript.url}`).default(this.publicAPI);
+                const m = await import(
+                    /* webpackInclude: /\.js$/ */
+                    /* webpackMode: "eager" */
+                    `./addons/${this.id}/${userscript.url}`
+                );
+                m.default(this.publicAPI);
             }
         }
     }
 
-    run () {
-        try {
-            this._run();
-        } catch (e) {
-            console.error('Addon error', e);
-        }
+    async run () {
+        this._run();
     }
 }
 AddonRunner.instances = [];

src/addons/pull.js

@@ -51,12 +51,10 @@ rimraf.sync('ScratchAddons');
 rimraf.sync('addons');
 rimraf.sync('addons-l10n');
 rimraf.sync('libraries');
-rimraf.sync('libraries-raw');
 childProcess.execSync('git clone --depth=1 -b tw https://github.com/GarboMuffin/ScratchAddons ScratchAddons');
 fs.mkdirSync('addons', {recursive: true});
 fs.mkdirSync('addons-l10n', {recursive: true});
 fs.mkdirSync('libraries', {recursive: true});
-fs.mkdirSync('libraries-raw', {recursive: true});
 
 const JS_HEADER = `/**!
  * Imported from SA
@@ -77,19 +75,6 @@ const includeImportedLibraries = contents => {
     }
 };
 
-const includeDynamicallyImportedLibraries = contents => {
-    // Parse things like:
-    // await addon.tab.loadScript(addon.self.lib + "/tinycolor-min.js");
-    const matches = [...contents.matchAll(/addon\.self\.lib *\+ *["']\/([\w\d_-]+\.js)["']/g)];
-    for (const match of matches) {
-        const libraryFile = match[1];
-        const oldLibraryPath = pathUtil.join('ScratchAddons', 'libraries', libraryFile);
-        const newLibraryPath = pathUtil.join('libraries-raw', libraryFile);
-        const libraryContents = fs.readFileSync(oldLibraryPath, 'utf-8');
-        fs.writeFileSync(newLibraryPath, libraryContents);
-    }
-};
-
 let fixHardcodedClassesCSS = '/* generated by pull.js */\n';
 const includeFixedHardcodedClasses = async contents => {
     // Parse things like:
@@ -158,6 +143,56 @@ const includeFixedHardcodedClasses = async contents => {
     }
 };
 
+const includeImports = (folder, contents) => {
+    // The first thing we have to do is figure out which files actually need to be loaded.
+    // Parse things like:
+    // await addon.tab.loadScript(addon.self.lib + "/tinycolor-min.js");
+    const matches = [...contents.matchAll(/addon\.self\.lib *\+ *["']\/([\w\d_-]+\.js)["']/g)];
+    const dynamicLibraries = [];
+    for (const match of matches) {
+        const libraryFile = match[1];
+        dynamicLibraries.push(libraryFile);
+        const oldLibraryPath = pathUtil.join('ScratchAddons', 'libraries', libraryFile);
+        const newLibraryPath = pathUtil.join('libraries', libraryFile);
+        const libraryContents = fs.readFileSync(oldLibraryPath, 'utf-8');
+        fs.writeFileSync(newLibraryPath, libraryContents);
+    }
+    const dynamicAssets = fs.readdirSync(folder)
+        .filter(file => file.endsWith('.svg'));
+
+    // Then we'll generate some JS to import them.
+    let header = '/* inserted by pull.js */\n';
+    dynamicAssets.forEach((file, index) => {
+        header += `import _twAsset${index} from "./${file}";\n`;
+    });
+    dynamicLibraries.forEach((file, index) => {
+        // Load as a file as it will be run through addon.tab.loadScript
+        header += `import _twScript${index} from "!file-loader!../../libraries/${file}";\n`;
+    });
+    header += `const _twGetAsset = (path) => {\n`;
+    dynamicAssets.forEach((file, index) => {
+        header += `  if (path === "/${file}") return _twAsset${index};\n`;
+    });
+    dynamicLibraries.forEach((file, index) => {
+        header += `  if (path === "/${file}") return _twScript${index};\n`;
+    });
+    header += '  throw new Error(`Unknown asset: ${path}`);\n';
+    header += '};\n';
+    header += '\n';
+
+    // And now we reroute everything to use our imports.
+    // Parse things like:
+    // el.src = addon.self.dir + "/" + name + ".svg";
+    //          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^  match
+    //                           ^^^^^^^^^^^^^^^^^^^  capture group 1
+    contents = contents.replace(
+        /addon\.self\.(?:dir|lib) *\+ *([^;]+)/g,
+        (_fullText, name) => `/* changed by pull.js */ _twGetAsset(${name})`
+    );
+
+    return header + contents;
+};
+
 (async () => {
     for (const addon of addons) {
         const oldDirectory = pathUtil.join('ScratchAddons', 'addons', addon);
@@ -168,17 +203,20 @@ const includeFixedHardcodedClasses = async contents => {
             fs.mkdirSync(newDirectory, {recursive: true});
             let contents = fs.readFileSync(oldPath, 'utf-8');
 
+            if (file.endsWith('.js')) {
+                includeImportedLibraries(contents);
+                // includeDynamicallyImportedLibraries(contents);
+                await includeFixedHardcodedClasses(contents);
+                if (contents.includes('addon.self.dir') || contents.includes('addon.self.lib')) {
+                    contents = includeImports(oldDirectory, contents);
+                }
+            }
+
             // Add a license notice, unless one already exists.
             if ((file.endsWith('.js') || file.endsWith('.css')) && !contents.includes('@license')) {
                 contents = JS_HEADER + contents;
             }
 
-            if (file.endsWith('.js')) {
-                includeImportedLibraries(contents);
-                includeDynamicallyImportedLibraries(contents);
-                await includeFixedHardcodedClasses(contents);
-            }
-
             fs.writeFileSync(newPath, contents);
         }
     }

webpack.config.js

@@ -22,9 +22,6 @@ const htmlWebpackPluginCommon = {
     root: root
 };
 
-const addonFolder = path.resolve(__dirname, 'src', 'addons', 'addons');
-const rawLibrariesFolder = path.resolve(__dirname, 'src', 'addons', 'libraries-raw');
-
 const base = {
     mode: process.env.NODE_ENV === 'production' ? 'production' : 'development',
     devtool: process.env.NODE_ENV === 'production' ? false : 'cheap-module-source-map',
@@ -66,9 +63,6 @@ const base = {
                 /node_modules[\\/]pify/,
                 /node_modules[\\/]@vernier[\\/]godirect/
             ],
-            exclude: [
-                rawLibrariesFolder
-            ],
             options: {
                 // Explicitly disable babelrc so we don't catch various config
                 // in much lower dependencies.
@@ -85,9 +79,6 @@ const base = {
         },
         {
             test: /\.css$/,
-            exclude: [
-                addonFolder
-            ],
             use: [{
                 loader: 'style-loader'
             }, {
@@ -111,24 +102,6 @@ const base = {
                     }
                 }
             }]
-        }, {
-            test: /\.css$/,
-            include: [
-                addonFolder
-            ],
-            loader: 'raw-loader'
-        },
-        {
-            test: /\.svg$/,
-            include: [
-                addonFolder
-            ],
-            loader: 'file-loader',
-            options: {
-                name: '[path][name].[ext]',
-                context: addonFolder,
-                outputPath: 'addon-files'
-            }
         }]
     },
     plugins: []
@@ -160,9 +133,6 @@ module.exports = [
                 {
                     test: /\.(svg|png|wav|gif|jpg|mp3)$/,
                     loader: 'file-loader',
-                    exclude: [
-                        addonFolder
-                    ],
                     options: {
                         outputPath: 'static/assets/'
                     }
@@ -244,10 +214,6 @@ module.exports = [
             new CopyWebpackPlugin([{
                 from: 'extension-worker.{js,js.map}',
                 context: 'node_modules/scratch-vm/dist/web'
-            }]),
-            new CopyWebpackPlugin([{
-                from: rawLibrariesFolder,
-                to: 'addon-files/libraries-raw'
             }])
         ])
     })
@@ -274,9 +240,6 @@ module.exports = [
                     {
                         test: /\.(svg|png|wav|gif|jpg|mp3)$/,
                         loader: 'file-loader',
-                        exclude: [
-                            addonFolder
-                        ],
                         options: {
                             outputPath: 'static/assets/',
                             publicPath: `${STATIC_PATH}/assets/`