ESM: Added new List<File> Dish type. Added present() method for displaying operation output in a nice way. Testing required.

src/core/Chef.mjs

@@ -91,8 +91,8 @@ class Chef {
 
         return {
             result: this.dish.type === Dish.HTML ?
-                this.dish.get(Dish.HTML, notUTF8) :
+                await this.dish.get(Dish.HTML, notUTF8) :
-                this.dish.get(returnType, notUTF8),
+                await this.dish.get(returnType, notUTF8),
             type: Dish.enumLookup(this.dish.type),
             progress: progress,
             duration: new Date().getTime() - startTime,

src/core/Dish.mjs

@@ -51,6 +51,8 @@ class Dish {
             case "bignumber":
             case "big number":
                 return Dish.BIG_NUMBER;
+            case "list<file>":
+                return Dish.LIST_FILE;
             default:
                 throw "Invalid data type string. No matching enum.";
         }
@@ -77,6 +79,8 @@ class Dish {
                 return "ArrayBuffer";
             case Dish.BIG_NUMBER:
                 return "BigNumber";
+            case Dish.LIST_FILE:
+                return "List<File>";
             default:
                 throw "Invalid data type enum. No matching type.";
         }
@@ -86,7 +90,7 @@ class Dish {
     /**
      * Sets the data value and type and then validates them.
      *
-     * @param {byteArray|string|number|ArrayBuffer|BigNumber} value
+     * @param {*} value
      *     - The value of the input data.
      * @param {number} type
      *     - The data type of value, see Dish enums.
@@ -112,15 +116,14 @@ class Dish {
      *
      * @param {number} type - The data type of value, see Dish enums.
      * @param {boolean} [notUTF8=false] - Do not treat strings as UTF8.
-     * @returns {byteArray|string|number|ArrayBuffer|BigNumber}
+     * @returns {*} - The value of the output data.
-     *     The value of the output data.
      */
-    get(type, notUTF8=false) {
+    async get(type, notUTF8=false) {
         if (typeof type === "string") {
             type = Dish.typeEnum(type);
         }
         if (this.type !== type) {
-            this.translate(type, notUTF8);
+            await this._translate(type, notUTF8);
         }
         return this.value;
     }
@@ -132,7 +135,7 @@ class Dish {
      * @param {number} toType - The data type of value, see Dish enums.
      * @param {boolean} [notUTF8=false] - Do not treat strings as UTF8.
      */
-    translate(toType, notUTF8=false) {
+    async _translate(toType, notUTF8=false) {
         log.debug(`Translating Dish from ${Dish.enumLookup(this.type)} to ${Dish.enumLookup(toType)}`);
         const byteArrayToStr = notUTF8 ? Utils.byteArrayToChars : Utils.byteArrayToUtf8;
 
@@ -142,7 +145,7 @@ class Dish {
                 this.value = this.value ? Utils.strToByteArray(this.value) : [];
                 break;
             case Dish.NUMBER:
-                this.value = typeof this.value == "number" ? Utils.strToByteArray(this.value.toString()) : [];
+                this.value = typeof this.value === "number" ? Utils.strToByteArray(this.value.toString()) : [];
                 break;
             case Dish.HTML:
                 this.value = this.value ? Utils.strToByteArray(Utils.unescapeHtml(Utils.stripHtmlTags(this.value, true))) : [];
@@ -154,6 +157,11 @@ class Dish {
             case Dish.BIG_NUMBER:
                 this.value = this.value instanceof BigNumber ? Utils.strToByteArray(this.value.toFixed()) : [];
                 break;
+            case Dish.LIST_FILE:
+                this.value = await Promise.all(this.value.map(async f => Utils.readFile(f)));
+                this.value = this.value.map(b => Array.prototype.slice.call(b));
+                this.value = [].concat.apply([], this.value);
+                break;
             default:
                 break;
         }
@@ -183,6 +191,10 @@ class Dish {
                 }
                 this.type = Dish.BIG_NUMBER;
                 break;
+            case Dish.LIST_FILE:
+                this.value = new File(this.value, "unknown");
+                this.type = Dish.LIST_FILE;
+                break;
             default:
                 break;
         }
@@ -220,6 +232,9 @@ class Dish {
                 return this.value instanceof ArrayBuffer;
             case Dish.BIG_NUMBER:
                 return this.value instanceof BigNumber;
+            case Dish.LIST_FILE:
+                return this.value instanceof Array &&
+                    this.value.reduce((acc, curr) => acc && curr instanceof File, true);
             default:
                 return false;
         }
@@ -244,6 +259,8 @@ class Dish {
                 return this.value.toString().length;
             case Dish.ARRAY_BUFFER:
                 return this.value.byteLength;
+            case Dish.LIST_FILE:
+                return this.value.reduce((acc, curr) => acc + curr.size, 0);
             default:
                 return -1;
         }
@@ -288,6 +305,12 @@ Dish.ARRAY_BUFFER = 4;
  * @enum
  */
 Dish.BIG_NUMBER = 5;
+/**
+ * Dish data type enum for lists of files.
+ * @readonly
+ * @enum
+ */
+Dish.LIST_FILE = 6;
 
 
 export default Dish;

src/core/FlowControl.js

@@ -26,7 +26,7 @@ const FlowControl = {
         const opList     = state.opList,
             inputType    = opList[state.progress].inputType,
             outputType   = opList[state.progress].outputType,
-            input        = state.dish.get(inputType),
+            input        = await state.dish.get(inputType),
             ings         = opList[state.progress].ingValues,
             splitDelim   = ings[0],
             mergeDelim   = ings[1],
@@ -77,7 +77,7 @@ const FlowControl = {
                 }
                 progress = err.progress + 1;
             }
-            output += dish.get(outputType) + mergeDelim;
+            output += await dish.get(outputType) + mergeDelim;
         }
 
         state.dish.set(output, outputType);
@@ -111,7 +111,7 @@ const FlowControl = {
      * @param {Operation[]} state.opList - The list of operations in the recipe.
      * @returns {Object} The updated state of the recipe.
      */
-    runRegister: function(state) {
+    runRegister: async function(state) {
         const ings = state.opList[state.progress].ingValues,
             extractorStr = ings[0],
             i = ings[1],
@@ -122,7 +122,7 @@ const FlowControl = {
         if (m) modifiers += "m";
 
         const extractor = new RegExp(extractorStr, modifiers),
-            input = state.dish.get(Dish.STRING),
+            input = await state.dish.get(Dish.STRING),
             registers = input.match(extractor);
 
         if (!registers) return state;
@@ -208,7 +208,7 @@ const FlowControl = {
      * @param {number} state.numJumps - The number of jumps taken so far.
      * @returns {Object} The updated state of the recipe.
      */
-    runCondJump: function(state) {
+    runCondJump: async function(state) {
         const ings     = state.opList[state.progress].ingValues,
             dish     = state.dish,
             regexStr = ings[0],
@@ -223,7 +223,7 @@ const FlowControl = {
         }
 
         if (regexStr !== "") {
-            const strMatch = dish.get(Dish.STRING).search(regexStr) > -1;
+            const strMatch = await dish.get(Dish.STRING).search(regexStr) > -1;
             if (!invert && strMatch || invert && !strMatch) {
                 state.progress = jmpIndex;
                 state.numJumps++;

src/core/Operation.mjs

@@ -19,6 +19,7 @@ class Operation {
         // Private fields
         this._inputType       = -1;
         this._outputType      = -1;
+        this._presentType     = -1;
         this._breakpoint      = false;
         this._disabled        = false;
         this._flowControl     = false;
@@ -71,6 +72,22 @@ class Operation {
     }
 
 
+    /**
+     * Method to be called when displaying the result of an operation in a human-readable
+     * format. This allows operations to return usable data from their run() method and
+     * only format them when this method is called.
+     *
+     * The default action is to return the data unchanged, but child classes can override
+     * this behaviour.
+     *
+     * @param {*} data - The result of the run() function
+     * @returns {*} - A human-readable version of the data
+     */
+    present(data) {
+        return data;
+    }
+
+
     /**
      * Sets the input type as a Dish enum.
      *
@@ -98,6 +115,7 @@ class Operation {
      */
     set outputType(typeStr) {
         this._outputType = Dish.typeEnum(typeStr);
+        if (this._presentType < 0) this._presentType = this._outputType;
     }
 
 
@@ -111,6 +129,26 @@ class Operation {
     }
 
 
+    /**
+     * Sets the presentation type as a Dish enum.
+     *
+     * @param {string} typeStr
+     */
+    set presentType(typeStr) {
+        this._presentType = Dish.typeEnum(typeStr);
+    }
+
+
+    /**
+     * Gets the presentation type as a readable string.
+     *
+     * @returns {string}
+     */
+    get presentType() {
+        return Dish.enumLookup(this._presentType);
+    }
+
+
     /**
      * Sets the args for the current operation.
      *

src/core/Recipe.mjs

@@ -130,7 +130,7 @@ class Recipe  {
      *     - The final progress through the recipe
      */
     async execute(dish, startFrom=0, forkState={}) {
-        let op, input, output,
+        let op, input, output, lastRunOp,
             numJumps = 0,
             numRegisters = forkState.numRegisters || 0;
 
@@ -149,7 +149,7 @@ class Recipe  {
             }
 
             try {
-                input = dish.get(op.inputType);
+                input = await dish.get(op.inputType);
                 log.debug("Executing operation");
 
                 if (op.flowControl) {
@@ -169,6 +169,7 @@ class Recipe  {
                     numRegisters = state.numRegisters;
                 } else {
                     output = await op.run(input, op.ingValues);
+                    lastRunOp = op;
                     dish.set(output, op.outputType);
                 }
             } catch (err) {
@@ -187,6 +188,11 @@ class Recipe  {
             }
         }
 
+        // Present the results of the final operation
+        // TODO try/catch
+        output = await lastRunOp.present(output);
+        dish.set(output, lastRunOp.presentType);
+
         log.debug("Recipe complete");
         return this.opList.length;
     }

src/core/Utils.mjs

@@ -812,35 +812,30 @@ class Utils {
 
     /**
      * Formats a list of files or directories.
-     * A File is an object with a "fileName" and optionally a "contents".
-     * If the fileName ends with "/" and the contents is of length 0 then
-     * it is considered a directory.
      *
      * @author tlwr [toby@toby.codes]
+     * @author n1474335 [n1474335@gmail.com]
      *
-     * @param {Object[]} files
+     * @param {File[]} files
      * @returns {html}
      */
-    static displayFilesAsHTML(files) {
+    static async displayFilesAsHTML(files) {
-        /* <NL> and <SP> used to denote newlines and spaces in HTML markup.
-         * If a non-html operation is used, all markup will be removed but these
-         * whitespace chars will remain for formatting purposes.
-         */
-
         const formatDirectory = function(file) {
             const html = `<div class='panel panel-default' style='white-space: normal;'>
                     <div class='panel-heading' role='tab'>
                         <h4 class='panel-title'>
-                            <NL>${Utils.escapeHtml(file.fileName)}
+                            ${Utils.escapeHtml(file.name)}
                         </h4>
                     </div>
                 </div>`;
             return html;
         };
 
-        const formatFile = function(file, i) {
+        const formatFile = async function(file, i) {
+            const buff = await Utils.readFile(file);
+            const fileStr = Utils.arrayBufferToStr(buff.buffer);
             const blob = new Blob(
-                [new Uint8Array(file.bytes)],
+                [buff],
                 {type: "octet/stream"}
             );
             const blobUrl = URL.createObjectURL(blob);
@@ -850,13 +845,13 @@ class Utils {
                 data-toggle='collapse'
                 aria-expanded='true'
                 aria-controls='collapse${i}'
-                title="Show/hide contents of '${Utils.escapeHtml(file.fileName)}'">&#x1f441;&#xfe0f;</a>`;
+                title="Show/hide contents of '${Utils.escapeHtml(file.name)}'">&#x1f441;&#xfe0f;</a>`;
 
             const downloadFileElem = `<a href='${blobUrl}'
-                title='Download ${Utils.escapeHtml(file.fileName)}'
+                title='Download ${Utils.escapeHtml(file.name)}'
-                download='${Utils.escapeHtml(file.fileName)}'>&#x1f4be;</a>`;
+                download='${Utils.escapeHtml(file.name)}'>&#x1f4be;</a>`;
 
-            const hexFileData = toHexFast(new Uint8Array(file.bytes));
+            const hexFileData = toHexFast(buff);
 
             const switchToInputElem = `<a href='#switchFileToInput${i}'
                 class='file-switch'
@@ -867,12 +862,12 @@ class Utils {
                     <div class='panel-heading' role='tab' id='heading${i}'>
                         <h4 class='panel-title'>
                             <div>
-                                ${Utils.escapeHtml(file.fileName)}<NL>
+                                ${Utils.escapeHtml(file.name)}
-                                ${viewFileElem}<SP>
+                                ${viewFileElem}
-                                ${downloadFileElem}<SP>
+                                ${downloadFileElem}
-                                ${switchToInputElem}<SP>
+                                ${switchToInputElem}
                                 <span class='pull-right'>
-                                    <NL>${file.size.toLocaleString()} bytes
+                                    ${file.size.toLocaleString()} bytes
                                 </span>
                             </div>
                         </h4>
@@ -880,7 +875,7 @@ class Utils {
                     <div id='collapse${i}' class='panel-collapse collapse'
                         role='tabpanel' aria-labelledby='heading${i}'>
                         <div class='panel-body'>
-                            <NL><NL><pre><code>${Utils.escapeHtml(file.contents)}</code></pre>
+                            <pre><code>${Utils.escapeHtml(fileStr)}</code></pre>
                         </div>
                     </div>
                 </div>`;
@@ -891,17 +886,15 @@ class Utils {
                 ${files.length} file(s) found<NL>
             </div>`;
 
-        files.forEach(function(file, i) {
+        for (let i = 0; i < files.length; i++) {
-            if (typeof file.contents !== "undefined") {
+            if (files[i].name.endsWith("/")) {
-                html += formatFile(file, i);
+                html += formatDirectory(files[i]);
             } else {
-                html += formatDirectory(file);
+                html += await formatFile(files[i], i);
             }
-        });
+        }
 
-        return html.replace(/(?:(<pre>(?:\n|.)*<\/pre>)|\s{2,})/g, "$1") // Remove whitespace from markup
+        return html;
-            .replace(/<NL>/g, "\n") // Replace <NP> with newlines
-            .replace(/<SP>/g, " "); // Replace <SP> with spaces
     }
 
 
@@ -941,6 +934,47 @@ class Utils {
     }
 
 
+    /**
+     * Reads a File and returns the data as a Uint8Array.
+     *
+     * @param {File} file
+     * @returns {Uint8Array}
+     *
+     * @example
+     * // returns Uint8Array(5) [104, 101, 108, 108, 111]
+     * await Utils.readFile(new File(["hello"], "test"))
+     */
+    static readFile(file) {
+        return new Promise((resolve, reject) => {
+            const reader = new FileReader();
+            const data = new Uint8Array(file.size);
+            let offset = 0;
+            const CHUNK_SIZE = 10485760; // 10MiB
+
+            const seek = function() {
+                if (offset >= file.size) {
+                    resolve(data);
+                    return;
+                }
+                const slice = file.slice(offset, offset + CHUNK_SIZE);
+                reader.readAsArrayBuffer(slice);
+            };
+
+            reader.onload = function(e) {
+                data.set(new Uint8Array(reader.result), offset);
+                offset += CHUNK_SIZE;
+                seek();
+            };
+
+            reader.onerror = function(e) {
+                reject(reader.error.message);
+            };
+
+            seek();
+        });
+    }
+
+
     /**
      * Actual modulo function, since % is actually the remainder function in JS.
      *

src/core/config/scripts/generateConfig.mjs

@@ -38,7 +38,7 @@ for (const opObj in Ops) {
         module: op.module,
         description: op.description,
         inputType: op.inputType,
-        outputType: op.outputType,
+        outputType: op.presentType,
         flowControl: op.flowControl,
         args: op.args
     };

src/core/operations/Unzip.mjs

@@ -25,7 +25,8 @@ class Unzip extends Operation {
         this.module = "Compression";
         this.description = "Decompresses data using the PKZIP algorithm and displays it per file, with support for passwords.";
         this.inputType = "byteArray";
-        this.outputType = "html";
+        this.outputType = "List<File>";
+        this.presentType = "html";
         this.args = [
             {
                 name: "Password",
@@ -43,7 +44,7 @@ class Unzip extends Operation {
     /**
      * @param {byteArray} input
      * @param {Object[]} args
-     * @returns {html}
+     * @returns {File[]}
      */
     run(input, args) {
         const options = {
@@ -51,28 +52,22 @@ class Unzip extends Operation {
                 verify: args[1]
             },
             unzip = new Zlib.Unzip(input, options),
-            filenames = unzip.getFilenames(),
+            filenames = unzip.getFilenames();
-            files = [];
 
-        filenames.forEach(function(fileName) {
+        return filenames.map(fileName => {
             const bytes = unzip.decompress(fileName);
-            const contents = Utils.byteArrayToUtf8(bytes);
+            return new File([bytes], fileName);
-
-            const file = {
-                fileName: fileName,
-                size: contents.length,
-            };
-
-            const isDir = contents.length === 0 && fileName.endsWith("/");
-            if (!isDir) {
-                file.bytes = bytes;
-                file.contents = contents;
-            }
-
-            files.push(file);
         });
+    }
 
-        return Utils.displayFilesAsHTML(files);
+    /**
+     * Displays the files in HTML for web apps.
+     *
+     * @param {File[]} files
+     * @returns {html}
+     */
+    async present(files) {
+        return await Utils.displayFilesAsHTML(files);
     }
 
 }