From f2d115ee4da1485dc821312db2562b344662f83c Mon Sep 17 00:00:00 2001 From: Klaxon Date: Sat, 29 Dec 2018 00:44:59 +1000 Subject: [PATCH 01/52] add lorem ipsum generator --- src/core/config/Categories.json | 3 +- src/core/lib/LoremIpsum.mjs | 221 ++++++++++++++++++++ src/core/operations/LoremIpsumGenerator.mjs | 70 +++++++ 3 files changed, 293 insertions(+), 1 deletion(-) create mode 100644 src/core/lib/LoremIpsum.mjs create mode 100644 src/core/operations/LoremIpsumGenerator.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 686c9842..3678a88b 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -375,7 +375,8 @@ "Parse QR Code", "Haversine distance", "Numberwang", - "XKCD Random Number" + "XKCD Random Number", + "Lorem Ipsum Generator" ] }, { diff --git a/src/core/lib/LoremIpsum.mjs b/src/core/lib/LoremIpsum.mjs new file mode 100644 index 00000000..9712a429 --- /dev/null +++ b/src/core/lib/LoremIpsum.mjs @@ -0,0 +1,221 @@ +/** + * Lorem Ipsum generator. + * + * @author Klaxon [klaxon@veyr.com] + * @copyright Crown Copyright 2016 + * @license Apache-2.0 + */ + + /** + * generate lorem ipsum paragraphs. + * + * @param {number} length + * @returns {string} + */ +export function GenerateParagraphs(length=3) { + const paragraphs = []; + while (paragraphs.length < length) { + const paragraphLength = getRandomLength(PARAGRAPH_LENGTH_MEAN, PARAGRAPH_LENGTH_STD_DEV); + const sentences = []; + while (sentences.length < paragraphLength) { + const sentenceLength = getRandomLength(SENTENCE_LENGTH_MEAN, SENTENCE_LENGTH_STD_DEV); + const sentence = getWords(sentenceLength); + sentences.push(formatSentence(sentence)); + } + paragraphs.push(formatParagraph(sentences)); + } + paragraphs[paragraphs.length-1] = paragraphs[paragraphs.length-1].slice(0, -2); + paragraphs[0] = replaceStart(paragraphs[0]); + return paragraphs.join(""); +} + +/** +* generate lorem ipsum sentences. +* +* @param {number} length +* @returns {string} +*/ +export function GenerateSentences(length=3) { + const sentences = []; + while (sentences.length < length) { + const sentenceLength = getRandomLength(SENTENCE_LENGTH_MEAN, SENTENCE_LENGTH_STD_DEV); + const sentence = getWords(sentenceLength); + sentences.push(formatSentence(sentence)); + } + const paragraphs = sentencesToParagraphs(sentences); + return paragraphs.join(""); +} + +/** +* generate lorem ipsum words. +* +* @param {number} length +* @returns {string} +*/ +export function GenerateWords(length=3) { + const words = getWords(length); + const sentences = wordsToSentences(words); + const paragraphs = sentencesToParagraphs(sentences); + return paragraphs.join(""); +} + + /** + * generate lorem ipsum bytes. + * + * @param {number} length + * @returns {string} + */ +export function GenerateBytes(length=3) { + const str = GenerateWords(length/3); + return str.slice(0, length); +} + +/** + * get array of randomly selected words from the lorem ipsum wordList. + * + * @param {number} length + * @returns {string[]} + * @private + */ +function getWords(length=3) { + const words = []; + let word; + let previousWord; + while (words.length < length){ + do { + word = wordList[Math.floor(Math.random() * wordList.length)]; + } + while (previousWord === word); + words.push(word); + previousWord = word; + } + return words; +} + +/** + * convert an array or words into an array of sentences" + * + * @param {string[]} words + * @returns {string[]} + * @private + */ +function wordsToSentences(words) { + const sentences = []; + while (words.length > 0) { + const sentenceLength = getRandomLength(SENTENCE_LENGTH_MEAN, SENTENCE_LENGTH_STD_DEV); + if (sentenceLength <= words.length) { + sentences.push(formatSentence(words.splice(0, sentenceLength))); + } else { + sentences.push(formatSentence(words.splice(0, words.length))); + } + } + return sentences; +} + +/** + * convert an array or sentences into an array of paragraphs" + * + * @param {string[]} sentences + * @returns {string[]} + * @private + */ +function sentencesToParagraphs(sentences) { + const paragraphs = []; + while (sentences.length > 0) { + const paragraphLength = getRandomLength(PARAGRAPH_LENGTH_MEAN, PARAGRAPH_LENGTH_STD_DEV); + paragraphs.push(formatParagraph(sentences.splice(0, paragraphLength))); + } + paragraphs[paragraphs.length-1] = paragraphs[paragraphs.length-1].slice(0, -1); + paragraphs[0] = replaceStart(paragraphs[0]); + return paragraphs; +} + +/** + * format an array of words into a sentence. + * + * @param {string[]} words + * @returns {string} + * @private + */ +function formatSentence(words) { + //0.35 chance of a comma being added randomly to the sentence. + if (Math.random() < PROBABILITY_OF_A_COMMA) { + const pos = Math.round(Math.random()*(words.length-1)); + words[pos] +=","; + } + let sentence = words.join(" "); + sentence = sentence.charAt(0).toUpperCase() + sentence.slice(1); + sentence += "."; + return sentence; +} + +/** + * format an array of sentences into a paragraph + * + * @param {string[]} sentences + * @returns {string} + * @private + */ +function formatParagraph(sentences) { + let paragraph = sentences.join(" "); + paragraph += "\n\n"; + return paragraph; +} + +/** + * get a random number based on a mean and standard deviation. + * + * @param {number} Mean + * @param {number} stdDev + * @returns {number} + * @private + */ +function getRandomLength(mean, stdDev) { + let length; + do { + length = Math.round((Math.random()*2-1)+(Math.random()*2-1)+(Math.random()*2-1)*stdDev+mean); + } + while (length <= 0); + return length; +} + +/** + * replace first 5 words with "Lorem ipsum dolor sit amet" + * + * @param {string[]} str + * @returns {string[]} + * @private + */ +function replaceStart(str) { + let words = str.split(" "); + if (words.length > 5) { + words.splice(0, 5, "Lorem", "ipsum", "dolor", "sit", "amet"); + return words.join(" "); + } else { + const lorem = ["Lorem", "ipsum", "dolor", "sit", "amet"]; + words = lorem.slice(0, words.length); + str = words.join(" "); + str += "."; + return str; + } +} + +const SENTENCE_LENGTH_MEAN = 15; +const SENTENCE_LENGTH_STD_DEV = 9; +const PARAGRAPH_LENGTH_MEAN = 5; +const PARAGRAPH_LENGTH_STD_DEV = 2; +const PROBABILITY_OF_A_COMMA = 0.35; + +const wordList = [ + "ad", "adipisicing", "aliqua", "aliquip", "amet", "anim", + "aute", "cillum", "commodo", "consectetur", "consequat", "culpa", + "cupidatat", "deserunt", "do", "dolor", "dolore", "duis", + "ea", "eiusmod", "elit", "enim", "esse", "est", + "et", "eu", "ex", "excepteur", "exercitation", "fugiat", + "id", "in", "incididunt", "ipsum", "irure", "labore", + "laboris", "laborum", "Lorem", "magna", "minim", "mollit", + "nisi", "non", "nostrud", "nulla", "occaecat", "officia", + "pariatur", "proident", "qui", "quis", "reprehenderit", "sint", + "sit", "sunt", "tempor", "ullamco", "ut", "velit", + "veniam", "voluptate", +]; diff --git a/src/core/operations/LoremIpsumGenerator.mjs b/src/core/operations/LoremIpsumGenerator.mjs new file mode 100644 index 00000000..228daaa1 --- /dev/null +++ b/src/core/operations/LoremIpsumGenerator.mjs @@ -0,0 +1,70 @@ +/** + * @author klaxon [klaxon@veyr.com] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import { GenerateParagraphs, GenerateSentences, GenerateWords, GenerateBytes } from "../lib/LoremIpsum"; + +/** + * Lorem Ipsum Generator operation + */ +class LoremIpsumGenerator extends Operation { + + /** + * LoremIpsumGenerator constructor + */ + constructor() { + super(); + + this.name = "Lorem Ipsum Generator"; + this.module = "Default"; + this.description = "Generate varying length lorem ipsum placeholder text."; + this.infoURL = "https://wikipedia.org/wiki/Lorem_ipsum"; + this.inputType = "string"; + this.outputType = "string"; + this.args = [ + { + "name": "Length", + "type": "number", + "value": "3" + }, + { + "name": "Length in", + "type": "option", + "value": ["Paragraphs", "Sentences", "Words", "Bytes"] + } + + ]; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + const [length, lengthType] = args; + if (length < 1){ + throw new OperationError("Length must be greater than 0"); + } + switch (lengthType) { + case "Paragraphs": + return GenerateParagraphs(length); + case "Sentences": + return GenerateSentences(length); + case "Words": + return GenerateWords(length); + case "Bytes": + return GenerateBytes(length); + default: + throw new OperationError("invalid lengthType"); + + } + } + +} + +export default LoremIpsumGenerator; From 32aea6b86cb982d4f901197c162ce10fe90ee644 Mon Sep 17 00:00:00 2001 From: Spencer Walden Date: Sun, 30 Dec 2018 03:20:24 -0800 Subject: [PATCH 02/52] Adds 'To Case Insensitive Regex' operation --- .../operations/ToCaseInsensitiveRegex.mjs | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 src/core/operations/ToCaseInsensitiveRegex.mjs diff --git a/src/core/operations/ToCaseInsensitiveRegex.mjs b/src/core/operations/ToCaseInsensitiveRegex.mjs new file mode 100644 index 00000000..32fb96c7 --- /dev/null +++ b/src/core/operations/ToCaseInsensitiveRegex.mjs @@ -0,0 +1,39 @@ +/** + * @author masq [github.cyberchef@masq.cc] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; + +/** + * To Case Insensitive Regex operation + */ +class ToCaseInsensitiveRegex extends Operation { + + /** + * ToCaseInsensitiveRegex constructor + */ + constructor() { + super(); + + this.name = "To Case Insensitive Regex"; + this.module = "Default"; + this.description = "Converts a case-sensitive regex string into a case-insensitive regex string in case /i flag is unavailable to you."; + this.infoURL = "https://wikipedia.org/wiki/Regular_expression"; + this.inputType = "string"; + this.outputType = "string"; + this.args = []; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + return input.replace(/[a-z]/ig, m => `[${m.toLowerCase()}${m.toUpperCase()}]`); + } +} + +export default ToCaseInsensitiveRegex; From 3c16b839b6fdbfe1d95cd0df6337f94d6f7f31c6 Mon Sep 17 00:00:00 2001 From: Spencer Walden Date: Sun, 30 Dec 2018 03:20:44 -0800 Subject: [PATCH 03/52] Adds 'From Case Insensitive Regex' operation --- .../operations/FromCaseInsensitiveRegex.mjs | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 src/core/operations/FromCaseInsensitiveRegex.mjs diff --git a/src/core/operations/FromCaseInsensitiveRegex.mjs b/src/core/operations/FromCaseInsensitiveRegex.mjs new file mode 100644 index 00000000..2448c5e5 --- /dev/null +++ b/src/core/operations/FromCaseInsensitiveRegex.mjs @@ -0,0 +1,39 @@ +/** + * @author masq [github.cyberchef@masq.cc] + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; + +/** + * From Case Insensitive Regex operation + */ +class FromCaseInsensitiveRegex extends Operation { + + /** + * FromCaseInsensitiveRegex constructor + */ + constructor() { + super(); + + this.name = "From Case Insensitive Regex"; + this.module = "Default"; + this.description = "Converts a case-insensitive regex string to a case sensitive regex string (no guarantee on it being the proper original casing) in case /i wasn't available at the time but now is, or you need it to be case-sensitive again."; + this.infoURL = "https://wikipedia.org/wiki/Regular_expression"; + this.inputType = "string"; + this.outputType = "string"; + this.args = []; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + return input.replace(/\[[a-z]{2}\]/ig, m => m[1].toUpperCase() === m[2].toUpperCase() ? m[1] : m); + } +} + +export default FromCaseInsensitiveRegex; From b750006cf0bef9b8609d30ffb445281150b0837a Mon Sep 17 00:00:00 2001 From: Spencer Walden Date: Sun, 30 Dec 2018 03:21:19 -0800 Subject: [PATCH 04/52] Adds tests for 'To/From Case Insensitive Regex' operations --- .../operations/ToFromInsensitiveRegex.mjs | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 test/tests/operations/ToFromInsensitiveRegex.mjs diff --git a/test/tests/operations/ToFromInsensitiveRegex.mjs b/test/tests/operations/ToFromInsensitiveRegex.mjs new file mode 100644 index 00000000..13c24e44 --- /dev/null +++ b/test/tests/operations/ToFromInsensitiveRegex.mjs @@ -0,0 +1,56 @@ +/** + * To/From Case Insensitive Regex tests. + * + * @author masq [github.cyberchef@masq.cc] + * + * @copyright Crown Copyright 2018 + * @license Apache-2.0 + */ +import TestRegister from "../../TestRegister"; + +TestRegister.addTests([ + { + name: "To Case Insensitive Regex: nothing", + input: "", + expectedOutput: "", + recipeConfig: [ + { + op: "To Case Insensitive Regex", + args: [], + }, + ], + }, + { + name: "From Case Insensitive Regex: nothing", + input: "", + expectedOutput: "", + recipeConfig: [ + { + op: "From Case Insensitive Regex", + args: [], + }, + ], + }, + { + name: "To Case Insensitive Regex: simple test", + input: "S0meth!ng", + expectedOutput: "[sS]0[mM][eE][tT][hH]![nN][gG]", + recipeConfig: [ + { + op: "To Case Insensitive Regex", + args: [], + }, + ], + }, + { + name: "From Case Insensitive Regex: simple test", + input: "[sS]0[mM][eE][tT][hH]![nN][Gg] [wr][On][g]?", + expectedOutput: "s0meth!nG [wr][On][g]?", + recipeConfig: [ + { + op: "From Case Insensitive Regex", + args: [], + }, + ], + }, +]); From 1d04b649e00f81220e8ad7c00744817d6aa87acc Mon Sep 17 00:00:00 2001 From: Spencer Walden Date: Sun, 30 Dec 2018 03:21:52 -0800 Subject: [PATCH 05/52] Adds 'To/From Case Insensitive Regex' operations under 'Utils' --- src/core/config/Categories.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index a93149fe..c2dad458 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -189,6 +189,8 @@ "Remove null bytes", "To Upper case", "To Lower case", + "To Case Insensitive Regex", + "From Case Insensitive Regex", "Add line numbers", "Remove line numbers", "To Table", From 126ad585c03f238769fc085fc151f56fd6fe1d2c Mon Sep 17 00:00:00 2001 From: Spencer Walden Date: Sun, 30 Dec 2018 03:22:23 -0800 Subject: [PATCH 06/52] Registers tests for 'To/From Case Insensitive Regex' operations --- test/index.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/index.mjs b/test/index.mjs index ff13fe3b..3900490a 100644 --- a/test/index.mjs +++ b/test/index.mjs @@ -82,6 +82,7 @@ import "./tests/operations/TranslateDateTimeFormat"; import "./tests/operations/Magic"; import "./tests/operations/ParseTLV"; import "./tests/operations/Media"; +import "./tests/operations/ToFromInsensitiveRegex"; let allTestsPassing = true; const testStatusCounts = { From 0f0e346a02bac4615ab3c7f168d65b2f3146dd8e Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 8 Jan 2019 11:12:02 +0000 Subject: [PATCH 07/52] Add new Subsection operation --- src/core/operations/Subsection.mjs | 141 +++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 src/core/operations/Subsection.mjs diff --git a/src/core/operations/Subsection.mjs b/src/core/operations/Subsection.mjs new file mode 100644 index 00000000..b9981100 --- /dev/null +++ b/src/core/operations/Subsection.mjs @@ -0,0 +1,141 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import XRegExp from "xregexp"; +import Operation from "../Operation"; +import Recipe from "../Recipe"; +import Dish from "../Dish"; + +/** + * Subsection operation + */ +class Subsection extends Operation { + + /** + * Subsection constructor + */ + constructor() { + super(); + + this.name = "Subsection"; + this.flowControl = true; + this.module = "Regex"; + this.description = "Select a part of the input data using regex, and run all subsequent operations on each match separately."; + this.infoURL = ""; + this.inputType = "string"; + this.outputType = "string"; + this.args = [ + { + "name": "Section (regex)", + "type": "string", + "value": "" + }, + { + "name": "Case sensitive matching", + "type": "boolean", + "value": true + }, + { + "name": "Ignore errors", + "type": "boolean", + "value": false + } + ]; + } + + /** + * @param {Object} state - The current state of the recipe. + * @param {number} state.progress - The current position in the recipe. + * @param {Dish} state.Dish - The Dish being operated on + * @param {Operation[]} state.opList - The list of operations in the recipe + * @returns {Object} - The updated state of the recipe + */ + async run(state) { + const opList = state.opList, + inputType = opList[state.progress].inputType, + outputType = opList[state.progress].outputType, + input = await state.dish.get(inputType), + ings = opList[state.progress].ingValues, + [section, caseSensitive, ignoreErrors] = ings, + subOpList = []; + + if (input && section !== "") { + // Create subOpList for each tranche to operate on + // (all remaining operations unless we encounter a Merge) + for (let i = state.progress + 1; i < opList.length; i++) { + if (opList[i].name === "Merge" && !opList[i].disabled) { + break; + } else { + subOpList.push(opList[i]); + } + } + + let flags = "g", + inOffset = 0, + output = "", + m, + progress = 0; + if (!caseSensitive) + flags += "i"; + const regex = new XRegExp(section, flags), + recipe = new Recipe(); + + recipe.addOperations(subOpList); + state.forkOffset += state.progress + 1; + + // Take a deep(ish) copy of the ingredient values + const ingValues = subOpList.map(op => JSON.parse(JSON.stringify(op.ingValues))); + let matched = false; + + // Run recipe over each match + while ((m = regex.exec(input))) { + matched = true; + // Add up to match + let matchStr = m[0]; + if (m.length === 1) { + output += input.slice(inOffset, m.index); + inOffset = regex.lastIndex; + } else if (m.length >= 2) { + matchStr = m[1]; + // Need to add some of the matched string that isn't in the capture group + output += input.slice(inOffset, m.index + m[0].indexOf(m[1])); + // Set i to be after the end of the first capture group + inOffset = regex.lastIndex - (m[0].length - (m[0].indexOf(m[1]) + m[1].length)); + } + // Baseline ing values for each tranche so that registers are reset + subOpList.forEach((op, i) => { + op.ingValues = JSON.parse(JSON.stringify(ingValues[i])); + }); + + const dish = new Dish(); + dish.set(matchStr, inputType); + + try { + progress = await recipe.execute(dish, 0, state); + } catch (err) { + if (!ignoreErrors) { + throw err; + } + progress = err.progress + 1; + } + output += await dish.get(outputType); + } + // If no matches were found, advance progress to after a Merge op + // Otherwise, the operations below Subsection will be run on all the input data + if (!matched) { + state.progress += subOpList.length + 1; + } + + output += input.slice(inOffset); + state.progress += progress; + state.dish.set(output, outputType); + } + return state; + } + +} + +export default Subsection; From 1a827ef44f034fc90c652e594fc97b4866935488 Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 8 Jan 2019 11:17:06 +0000 Subject: [PATCH 08/52] Add Subsection to Flow Control category --- src/core/config/Categories.json | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index e9fe3399..832c91e8 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -364,6 +364,7 @@ "ops": [ "Magic", "Fork", + "Subsection", "Merge", "Register", "Label", From 8ac5b484938fe86da5fc332f0e6c4f686ae73db1 Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 8 Jan 2019 11:51:33 +0000 Subject: [PATCH 09/52] Update operation description --- src/core/operations/Subsection.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/Subsection.mjs b/src/core/operations/Subsection.mjs index b9981100..94258b6d 100644 --- a/src/core/operations/Subsection.mjs +++ b/src/core/operations/Subsection.mjs @@ -23,7 +23,7 @@ class Subsection extends Operation { this.name = "Subsection"; this.flowControl = true; this.module = "Regex"; - this.description = "Select a part of the input data using regex, and run all subsequent operations on each match separately."; + this.description = "Select a part of the input data using a regular expression (regex), and run all subsequent operations on each match separately.

You can use up to one capture group, where the recipe will only be run on the data in the capture group. If there's more than one capture group, only the first one will be operated on."; this.infoURL = ""; this.inputType = "string"; this.outputType = "string"; From 5ac469b17467f8f279d592b34e00dd629227f95e Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 8 Jan 2019 16:19:58 +0000 Subject: [PATCH 10/52] Added yara rule support --- package-lock.json | 151 +++++++++++++++--------------- package.json | 1 + src/core/operations/YaraRules.mjs | 68 ++++++++++++++ 3 files changed, 147 insertions(+), 73 deletions(-) create mode 100644 src/core/operations/YaraRules.mjs diff --git a/package-lock.json b/package-lock.json index 4f1be0a3..f1980a00 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1171,7 +1171,7 @@ }, "ansi-escapes": { "version": "3.1.0", - "resolved": "http://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", "integrity": "sha512-UgAb8H9D41AQnu/PbWlCofQVcnV4Gs2bBJi9eZPxfU/hgglFh3SMDMENRIqdr7H6XFnXdoknctFByVsCOotTVw==", "dev": true }, @@ -1284,7 +1284,7 @@ }, "array-equal": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=", "dev": true }, @@ -1369,7 +1369,7 @@ }, "util": { "version": "0.10.3", - "resolved": "http://registry.npmjs.org/util/-/util-0.10.3.tgz", + "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", "dev": true, "requires": { @@ -1457,7 +1457,7 @@ }, "axios": { "version": "0.18.0", - "resolved": "http://registry.npmjs.org/axios/-/axios-0.18.0.tgz", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz", "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=", "dev": true, "requires": { @@ -1863,7 +1863,7 @@ }, "browserify-aes": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", "dev": true, "requires": { @@ -1900,7 +1900,7 @@ }, "browserify-rsa": { "version": "4.0.1", - "resolved": "http://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", + "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=", "dev": true, "requires": { @@ -1950,7 +1950,7 @@ }, "buffer": { "version": "4.9.1", - "resolved": "http://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", "dev": true, "requires": { @@ -2015,7 +2015,7 @@ }, "cacache": { "version": "10.0.4", - "resolved": "http://registry.npmjs.org/cacache/-/cacache-10.0.4.tgz", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-10.0.4.tgz", "integrity": "sha512-Dph0MzuH+rTQzGPNT9fAnrPmMmjKfST6trxJeK7NQuHRaVw24VzPRWTmg9MpcwOVQZO0E1FBICUlFeNaKPIfHA==", "dev": true, "requires": { @@ -2092,7 +2092,7 @@ }, "camelcase-keys": { "version": "2.1.0", - "resolved": "http://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=", "dev": true, "requires": { @@ -2123,7 +2123,7 @@ }, "chalk": { "version": "1.1.3", - "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", "requires": { "ansi-styles": "^2.2.1", @@ -2590,7 +2590,7 @@ }, "create-hash": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", "dev": true, "requires": { @@ -2603,7 +2603,7 @@ }, "create-hmac": { "version": "1.1.7", - "resolved": "http://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", + "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", "dev": true, "requires": { @@ -2721,7 +2721,7 @@ }, "css-select": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", "dev": true, "requires": { @@ -3055,7 +3055,7 @@ }, "diffie-hellman": { "version": "5.0.3", - "resolved": "http://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", + "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==", "dev": true, "requires": { @@ -3119,7 +3119,7 @@ "dependencies": { "domelementtype": { "version": "1.1.3", - "resolved": "http://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=", "dev": true }, @@ -3307,7 +3307,7 @@ }, "entities": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/entities/-/entities-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.0.0.tgz", "integrity": "sha1-sph6o4ITR/zeZCsk/fyeT7cSvyY=", "dev": true }, @@ -3731,7 +3731,7 @@ }, "eventemitter2": { "version": "0.4.14", - "resolved": "http://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", + "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", "integrity": "sha1-j2G3XN4BKy6esoTUVFWDtWQ7Yas=", "dev": true }, @@ -3743,7 +3743,7 @@ }, "events": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/events/-/events-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", "dev": true }, @@ -4149,7 +4149,7 @@ }, "finalhandler": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", "dev": true, "requires": { @@ -4377,7 +4377,7 @@ }, "fs-extra": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", "integrity": "sha1-zTzl9+fLYUWIP8rjGR6Yd/hYeVA=", "dev": true, "requires": { @@ -5023,7 +5023,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", "dev": true }, @@ -5103,7 +5103,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -5173,7 +5173,7 @@ }, "grunt-cli": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/grunt-cli/-/grunt-cli-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/grunt-cli/-/grunt-cli-1.2.0.tgz", "integrity": "sha1-VisRnrsGndtGSs4oRVAb6Xs1tqg=", "dev": true, "requires": { @@ -5221,7 +5221,7 @@ "dependencies": { "shelljs": { "version": "0.5.3", - "resolved": "http://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz", "integrity": "sha1-xUmCuZbHbvDB5rWfvcWCX1txMRM=", "dev": true } @@ -5241,7 +5241,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -5269,7 +5269,7 @@ }, "grunt-contrib-jshint": { "version": "1.1.0", - "resolved": "http://registry.npmjs.org/grunt-contrib-jshint/-/grunt-contrib-jshint-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/grunt-contrib-jshint/-/grunt-contrib-jshint-1.1.0.tgz", "integrity": "sha1-Np2QmyWTxA6L55lAshNAhQx5Oaw=", "dev": true, "requires": { @@ -5368,7 +5368,7 @@ "dependencies": { "colors": { "version": "1.1.2", - "resolved": "http://registry.npmjs.org/colors/-/colors-1.1.2.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz", "integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM=", "dev": true } @@ -5432,7 +5432,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -5450,7 +5450,7 @@ }, "handle-thing": { "version": "1.2.5", - "resolved": "http://registry.npmjs.org/handle-thing/-/handle-thing-1.2.5.tgz", + "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-1.2.5.tgz", "integrity": "sha1-/Xqtcmvxpf0W38KbL3pmAdJxOcQ=", "dev": true }, @@ -5725,7 +5725,7 @@ }, "htmlparser2": { "version": "3.8.3", - "resolved": "http://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", "integrity": "sha1-mWwosZFRaovoZQGn15dX5ccMEGg=", "dev": true, "requires": { @@ -5744,7 +5744,7 @@ }, "http-errors": { "version": "1.6.3", - "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", "dev": true, "requires": { @@ -5773,7 +5773,7 @@ }, "http-proxy-middleware": { "version": "0.18.0", - "resolved": "http://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", "integrity": "sha512-Fs25KVMPAIIcgjMZkVHJoKg9VcXcC1C8yb9JUgeDvVXY0S/zgVIhMb+qVswDIgtJe2DfckMSY2d6TuTEutlk6Q==", "dev": true, "requires": { @@ -6225,7 +6225,7 @@ }, "is-builtin-module": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=", "dev": true, "requires": { @@ -6750,7 +6750,7 @@ }, "jsonfile": { "version": "2.4.0", - "resolved": "http://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=", "dev": true, "requires": { @@ -6856,7 +6856,7 @@ }, "kew": { "version": "0.7.0", - "resolved": "http://registry.npmjs.org/kew/-/kew-0.7.0.tgz", + "resolved": "https://registry.npmjs.org/kew/-/kew-0.7.0.tgz", "integrity": "sha1-edk9LTM2PW/dKXCzNdkUGtWR15s=", "dev": true }, @@ -6928,6 +6928,11 @@ "resolved": "https://registry.npmjs.org/lex-parser/-/lex-parser-0.1.4.tgz", "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA=" }, + "libyara-wasm": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.2.tgz", + "integrity": "sha512-Q4Biyfwiwrz7u25j7HNbGjJkcuekMJJeGCwbUgc7s7GMriZBSctCJPzlP5MBKgMzGMe0fNc9Hu5EP6Y1lBFnWg==" + }, "livereload-js": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/livereload-js/-/livereload-js-2.4.0.tgz", @@ -6948,7 +6953,7 @@ }, "load-json-file": { "version": "1.1.0", - "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", "dev": true, "requires": { @@ -6961,7 +6966,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -7196,7 +7201,7 @@ }, "media-typer": { "version": "0.3.0", - "resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", "dev": true }, @@ -7255,7 +7260,7 @@ }, "meow": { "version": "3.7.0", - "resolved": "http://registry.npmjs.org/meow/-/meow-3.7.0.tgz", + "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz", "integrity": "sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=", "dev": true, "requires": { @@ -7432,7 +7437,7 @@ }, "mkdirp": { "version": "0.5.1", - "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "dev": true, "requires": { @@ -7554,7 +7559,7 @@ }, "ncp": { "version": "1.0.1", - "resolved": "http://registry.npmjs.org/ncp/-/ncp-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/ncp/-/ncp-1.0.1.tgz", "integrity": "sha1-0VNn5cuHQyuhF9K/gP30Wuz7QkY=", "dev": true }, @@ -7617,7 +7622,7 @@ "dependencies": { "semver": { "version": "5.3.0", - "resolved": "http://registry.npmjs.org/semver/-/semver-5.3.0.tgz", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=", "dev": true } @@ -7756,7 +7761,7 @@ "dependencies": { "colors": { "version": "0.5.1", - "resolved": "http://registry.npmjs.org/colors/-/colors-0.5.1.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-0.5.1.tgz", "integrity": "sha1-fQAj6usVTo7p/Oddy5I9DtFmd3Q=" }, "underscore": { @@ -8015,13 +8020,13 @@ }, "os-homedir": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", "dev": true }, "os-locale": { "version": "1.4.0", - "resolved": "http://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=", "dev": true, "requires": { @@ -8030,7 +8035,7 @@ }, "os-tmpdir": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "dev": true }, @@ -8173,7 +8178,7 @@ }, "parse-asn1": { "version": "5.1.1", - "resolved": "http://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", + "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", "integrity": "sha512-KPx7flKXg775zZpnp9SxJlz00gTd4BmJ2yJufSc44gMCRrRQ7NSzAcSJQfifuOLgW6bEi+ftrALtsgALeB2Adw==", "dev": true, "requires": { @@ -8231,7 +8236,7 @@ }, "path-is-absolute": { "version": "1.0.1", - "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true }, @@ -8272,7 +8277,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -8437,7 +8442,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -8839,7 +8844,7 @@ }, "progress": { "version": "1.1.8", - "resolved": "http://registry.npmjs.org/progress/-/progress-1.1.8.tgz", + "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz", "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=" }, "promise-inflight": { @@ -8864,13 +8869,13 @@ "dependencies": { "async": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=", "dev": true }, "winston": { "version": "2.1.1", - "resolved": "http://registry.npmjs.org/winston/-/winston-2.1.1.tgz", + "resolved": "https://registry.npmjs.org/winston/-/winston-2.1.1.tgz", "integrity": "sha1-PJNJ0ZYgf9G9/51LxD73JRDjoS4=", "dev": true, "requires": { @@ -8885,7 +8890,7 @@ "dependencies": { "colors": { "version": "1.0.3", - "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true }, @@ -9064,7 +9069,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -9253,7 +9258,7 @@ "dependencies": { "jsesc": { "version": "0.5.0", - "resolved": "http://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", "integrity": "sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0=", "dev": true } @@ -9304,7 +9309,7 @@ }, "htmlparser2": { "version": "3.3.0", - "resolved": "http://registry.npmjs.org/htmlparser2/-/htmlparser2-3.3.0.tgz", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.3.0.tgz", "integrity": "sha1-zHDQWln2VC5D8OaFyYLhTJJKnv4=", "dev": true, "requires": { @@ -9316,7 +9321,7 @@ }, "readable-stream": { "version": "1.0.34", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", "dev": true, "requires": { @@ -9426,7 +9431,7 @@ }, "require-uncached": { "version": "1.0.3", - "resolved": "http://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", "integrity": "sha1-Tg1W1slmL9MeQwEcS5WqSZVUIdM=", "dev": true, "requires": { @@ -9593,7 +9598,7 @@ }, "safe-regex": { "version": "1.1.0", - "resolved": "http://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=", "dev": true, "requires": { @@ -9914,7 +9919,7 @@ }, "sha.js": { "version": "2.4.11", - "resolved": "http://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", + "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", "dev": true, "requires": { @@ -9958,7 +9963,7 @@ }, "shelljs": { "version": "0.3.0", - "resolved": "http://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", "integrity": "sha1-NZbmMHp4FUT1kfN9phg2DzHbV7E=", "dev": true }, @@ -10610,7 +10615,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "requires": { "ansi-regex": "^2.0.0" @@ -10627,7 +10632,7 @@ }, "strip-eof": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", "dev": true }, @@ -10706,7 +10711,7 @@ }, "tar": { "version": "2.2.1", - "resolved": "http://registry.npmjs.org/tar/-/tar-2.2.1.tgz", + "resolved": "https://registry.npmjs.org/tar/-/tar-2.2.1.tgz", "integrity": "sha1-jk0qJWwOIYXGsYrWlK7JaLg8sdE=", "dev": true, "requires": { @@ -10734,7 +10739,7 @@ }, "through": { "version": "2.3.8", - "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", "dev": true }, @@ -11381,7 +11386,7 @@ "dependencies": { "async": { "version": "0.9.2", - "resolved": "http://registry.npmjs.org/async/-/async-0.9.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=", "dev": true }, @@ -11407,7 +11412,7 @@ }, "valid-data-url": { "version": "0.1.6", - "resolved": "http://registry.npmjs.org/valid-data-url/-/valid-data-url-0.1.6.tgz", + "resolved": "https://registry.npmjs.org/valid-data-url/-/valid-data-url-0.1.6.tgz", "integrity": "sha512-FXg2qXMzfAhZc0y2HzELNfUeiOjPr+52hU1DNBWiJJ2luXD+dD1R9NA48Ug5aj0ibbxroeGDc/RJv6ThiGgkDw==", "dev": true }, @@ -11423,7 +11428,7 @@ }, "validator": { "version": "9.4.1", - "resolved": "http://registry.npmjs.org/validator/-/validator-9.4.1.tgz", + "resolved": "https://registry.npmjs.org/validator/-/validator-9.4.1.tgz", "integrity": "sha512-YV5KjzvRmSyJ1ee/Dm5UED0G+1L4GZnLN3w6/T+zZm8scVua4sOhYKWTUrKa0H/tMiJyO9QLHMPN+9mB/aMunA==", "dev": true }, @@ -11847,7 +11852,7 @@ }, "webpack-node-externals": { "version": "1.7.2", - "resolved": "http://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.7.2.tgz", + "resolved": "https://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.7.2.tgz", "integrity": "sha512-ajerHZ+BJKeCLviLUUmnyd5B4RavLF76uv3cs6KNuO8W+HuQaEs0y0L7o40NQxdPy5w0pcv8Ew7yPUAQG0UdCg==", "dev": true }, @@ -11944,14 +11949,14 @@ "dependencies": { "async": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=", "dev": true, "optional": true }, "colors": { "version": "1.0.3", - "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true, "optional": true @@ -11984,7 +11989,7 @@ }, "wrap-ansi": { "version": "2.1.0", - "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", "dev": true, "requires": { diff --git a/package.json b/package.json index 4fac8034..8c0740ca 100644 --- a/package.json +++ b/package.json @@ -101,6 +101,7 @@ "jsonwebtoken": "^8.3.0", "jsrsasign": "8.0.12", "kbpgp": "^2.0.82", + "libyara-wasm": "0.0.2", "lodash": "^4.17.11", "loglevel": "^1.6.1", "loglevel-message-prefix": "^3.0.0", diff --git a/src/core/operations/YaraRules.mjs b/src/core/operations/YaraRules.mjs new file mode 100644 index 00000000..9f2d9b79 --- /dev/null +++ b/src/core/operations/YaraRules.mjs @@ -0,0 +1,68 @@ +/** + * @author Matt C [matt@artemisbot.uk] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import Yara from "libyara-wasm"; + +/** + * Yara Rules operation + */ +class YaraRules extends Operation { + + /** + * YaraRules constructor + */ + constructor() { + super(); + + this.name = "Yara Rules"; + this.module = "Yara"; + this.description = "Yara support"; + this.infoURL = "https://en.wikipedia.org/wiki/YARA"; + this.inputType = "string"; + this.outputType = "string"; + this.args = [{ + name: "Rules", + type: "string", + value: "" + }]; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + return new Promise((resolve, reject) => { + Yara().then(yara => { + const resp = yara.run(input, args[0]); + if (resp.compileErrors.size() > 0) { + for (let i = 0; i < resp.compileErrors.size(); i++) { + const compileError = resp.compileErrors.get(i); + reject(new OperationError(`Error on line ${compileError.lineNumber}: ${compileError.message}`)); + } + } + const matchedRules = resp.matchedRules; + let matchString = ""; + for (let i = 0; i < matchedRules.keys().size(); i++) { + const ruleMatches = matchedRules.get(matchedRules.keys().get(i)); + matchString += `Rule "${matchedRules.keys().get(i)}" matches:\n`; + + for (let j = 0; j < ruleMatches.size(); j++) { + const match = ruleMatches.get(j); + matchString += `Position ${match.location}, length ${match.matchLength}, data: ${match.data}\n`; + } + } + resolve(matchString); + }); + }); + } + +} + +export default YaraRules; From 766de7e6fa2475ab4d3a1d2d5ce5f1f5f6862ab0 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 8 Jan 2019 17:51:43 +0000 Subject: [PATCH 11/52] Fixed bug in 'Regular expression' operation when highlighting lookaheads --- package-lock.json | 128 +++++++++++----------- src/core/operations/RegularExpression.mjs | 31 ++---- 2 files changed, 76 insertions(+), 83 deletions(-) diff --git a/package-lock.json b/package-lock.json index 57175720..d19abbcc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1497,7 +1497,7 @@ }, "ansi-escapes": { "version": "3.1.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", + "resolved": "http://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", "integrity": "sha512-UgAb8H9D41AQnu/PbWlCofQVcnV4Gs2bBJi9eZPxfU/hgglFh3SMDMENRIqdr7H6XFnXdoknctFByVsCOotTVw==", "dev": true }, @@ -1571,7 +1571,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -1615,7 +1615,7 @@ }, "array-equal": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=", "dev": true }, @@ -1700,7 +1700,7 @@ }, "util": { "version": "0.10.3", - "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", + "resolved": "http://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", "dev": true, "requires": { @@ -1848,7 +1848,7 @@ }, "axios": { "version": "0.18.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz", + "resolved": "http://registry.npmjs.org/axios/-/axios-0.18.0.tgz", "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=", "dev": true, "requires": { @@ -2286,7 +2286,7 @@ }, "browserify-aes": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", "dev": true, "requires": { @@ -2323,7 +2323,7 @@ }, "browserify-rsa": { "version": "4.0.1", - "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", + "resolved": "http://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=", "dev": true, "requires": { @@ -2388,7 +2388,7 @@ }, "buffer": { "version": "4.9.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "resolved": "http://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", "dev": true, "requires": { @@ -2551,7 +2551,7 @@ }, "camelcase-keys": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", + "resolved": "http://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=", "dev": true, "requires": { @@ -2600,7 +2600,7 @@ }, "chalk": { "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", "requires": { "ansi-styles": "^2.2.1", @@ -2779,7 +2779,7 @@ }, "string-width": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "dev": true, "requires": { @@ -3133,7 +3133,7 @@ }, "create-hash": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", "dev": true, "requires": { @@ -3146,7 +3146,7 @@ }, "create-hmac": { "version": "1.1.7", - "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", + "resolved": "http://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", "dev": true, "requires": { @@ -3322,7 +3322,7 @@ }, "regexpu-core": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/regexpu-core/-/regexpu-core-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-1.0.0.tgz", "integrity": "sha1-hqdj9Y7k18L2sQLkdkBQ3n7ZDGs=", "dev": true, "requires": { @@ -3339,7 +3339,7 @@ }, "regjsparser": { "version": "0.1.5", - "resolved": "http://registry.npmjs.org/regjsparser/-/regjsparser-0.1.5.tgz", + "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.1.5.tgz", "integrity": "sha1-fuj4Tcb6eS0/0K4ijSS9lJ6tIFw=", "dev": true, "requires": { @@ -4271,7 +4271,7 @@ }, "source-map": { "version": "0.1.43", - "resolved": "http://registry.npmjs.org/source-map/-/source-map-0.1.43.tgz", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.1.43.tgz", "integrity": "sha1-wkvBRspRfBRx9drL4lcbK3+eM0Y=", "requires": { "amdefine": ">=0.0.4" @@ -4461,7 +4461,7 @@ "dependencies": { "source-map": { "version": "0.5.0", - "resolved": "http://registry.npmjs.org/source-map/-/source-map-0.5.0.tgz", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.0.tgz", "integrity": "sha1-D+llA6yGpa213mP05BKuSHLNvoY=", "dev": true } @@ -4794,7 +4794,7 @@ }, "finalhandler": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", "dev": true, "requires": { @@ -4829,7 +4829,7 @@ }, "findup-sync": { "version": "0.3.0", - "resolved": "http://registry.npmjs.org/findup-sync/-/findup-sync-0.3.0.tgz", + "resolved": "https://registry.npmjs.org/findup-sync/-/findup-sync-0.3.0.tgz", "integrity": "sha1-N5MKpdgWt3fANEXhlmzGeQpMCxY=", "dev": true, "requires": { @@ -4896,7 +4896,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -5019,7 +5019,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -5098,12 +5098,14 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5123,7 +5125,8 @@ "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", @@ -5271,6 +5274,7 @@ "version": "3.0.4", "bundled": true, "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5660,7 +5664,7 @@ }, "string-width": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "dev": true, "requires": { @@ -5823,7 +5827,7 @@ }, "globby": { "version": "6.1.0", - "resolved": "http://registry.npmjs.org/globby/-/globby-6.1.0.tgz", + "resolved": "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz", "integrity": "sha1-9abXDoOV4hyFj7BInWTfAkJNUGw=", "dev": true, "requires": { @@ -5935,7 +5939,7 @@ }, "resolve": { "version": "1.1.7", - "resolved": "http://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz", "integrity": "sha1-IDEU2CrSxe2ejgQRs5ModeiJ6Xs=", "dev": true } @@ -5981,7 +5985,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -6125,7 +6129,7 @@ "dependencies": { "colors": { "version": "1.1.2", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-1.1.2.tgz", "integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM=", "dev": true } @@ -6189,7 +6193,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -6409,7 +6413,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -6970,7 +6974,7 @@ }, "is-accessor-descriptor": { "version": "0.1.6", - "resolved": "http://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", "dev": true, "requires": { @@ -7025,7 +7029,7 @@ }, "is-data-descriptor": { "version": "0.1.4", - "resolved": "http://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", "dev": true, "requires": { @@ -7429,7 +7433,7 @@ }, "underscore": { "version": "1.8.3", - "resolved": "http://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz", "integrity": "sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI=", "dev": true } @@ -8222,7 +8226,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -8471,7 +8475,7 @@ "dependencies": { "commander": { "version": "2.15.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz", + "resolved": "http://registry.npmjs.org/commander/-/commander-2.15.1.tgz", "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==", "dev": true, "optional": true @@ -8591,7 +8595,7 @@ }, "multimatch": { "version": "2.1.0", - "resolved": "http://registry.npmjs.org/multimatch/-/multimatch-2.1.0.tgz", + "resolved": "https://registry.npmjs.org/multimatch/-/multimatch-2.1.0.tgz", "integrity": "sha1-nHkGoi+0wCkZ4vX3UWG0zb1LKis=", "dev": true, "requires": { @@ -8603,7 +8607,7 @@ }, "mute-stream": { "version": "0.0.7", - "resolved": "http://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", "dev": true }, @@ -8922,12 +8926,12 @@ "dependencies": { "colors": { "version": "0.5.1", - "resolved": "https://registry.npmjs.org/colors/-/colors-0.5.1.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-0.5.1.tgz", "integrity": "sha1-fQAj6usVTo7p/Oddy5I9DtFmd3Q=" }, "underscore": { "version": "1.1.7", - "resolved": "http://registry.npmjs.org/underscore/-/underscore-1.1.7.tgz", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.1.7.tgz", "integrity": "sha1-QLq4S60Z0jAJbo1u9ii/8FXYPbA=" } } @@ -9426,7 +9430,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -9514,7 +9518,7 @@ }, "path-browserify": { "version": "0.0.0", - "resolved": "http://registry.npmjs.org/path-browserify/-/path-browserify-0.0.0.tgz", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-0.0.0.tgz", "integrity": "sha1-oLhwcpquIUAFt9UDLsLLuw+0RRo=", "dev": true }, @@ -10152,13 +10156,13 @@ "dependencies": { "async": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz", "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=", "dev": true }, "winston": { "version": "2.1.1", - "resolved": "https://registry.npmjs.org/winston/-/winston-2.1.1.tgz", + "resolved": "http://registry.npmjs.org/winston/-/winston-2.1.1.tgz", "integrity": "sha1-PJNJ0ZYgf9G9/51LxD73JRDjoS4=", "dev": true, "requires": { @@ -10173,7 +10177,7 @@ "dependencies": { "colors": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true }, @@ -10490,7 +10494,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -10783,7 +10787,7 @@ "dependencies": { "underscore": { "version": "1.6.0", - "resolved": "http://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag=", "dev": true } @@ -11102,7 +11106,7 @@ "dependencies": { "source-map": { "version": "0.4.4", - "resolved": "http://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=", "dev": true, "requires": { @@ -11710,7 +11714,7 @@ }, "split2": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/split2/-/split2-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/split2/-/split2-1.1.1.tgz", "integrity": "sha1-Fi2bGIZfAqsvKtlYVSLbm1TEgfk=", "dev": true, "requires": { @@ -11719,7 +11723,7 @@ }, "sprintf-js": { "version": "1.0.3", - "resolved": "http://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", "dev": true }, @@ -11851,7 +11855,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -11868,7 +11872,7 @@ }, "stream-browserify": { "version": "2.0.1", - "resolved": "http://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.1.tgz", + "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.1.tgz", "integrity": "sha1-ZiZu5fm9uZQKTkUUyvtDu3Hlyds=", "dev": true, "requires": { @@ -11954,7 +11958,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -12071,7 +12075,7 @@ }, "supports-color": { "version": "2.0.0", - "resolved": "http://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" }, "symbol-tree": { @@ -12534,7 +12538,7 @@ }, "tty-browserify": { "version": "0.0.0", - "resolved": "http://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz", + "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz", "integrity": "sha1-oVe6QC2iTpv5V/mqadUk7tQpAaY=", "dev": true }, @@ -12609,7 +12613,7 @@ }, "underscore": { "version": "1.7.0", - "resolved": "http://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz", "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=" }, "underscore-contrib": { @@ -12623,7 +12627,7 @@ "dependencies": { "underscore": { "version": "1.6.0", - "resolved": "http://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag=", "dev": true } @@ -12938,7 +12942,7 @@ "dependencies": { "async": { "version": "0.9.2", - "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-0.9.2.tgz", "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=", "dev": true }, @@ -13008,7 +13012,7 @@ }, "vm-browserify": { "version": "0.0.4", - "resolved": "http://registry.npmjs.org/vm-browserify/-/vm-browserify-0.0.4.tgz", + "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-0.0.4.tgz", "integrity": "sha1-XX6kW7755Kb/ZflUOOCofDV9WnM=", "dev": true, "requires": { @@ -13666,14 +13670,14 @@ "dependencies": { "async": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz", "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=", "dev": true, "optional": true }, "colors": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true, "optional": true @@ -13725,7 +13729,7 @@ }, "string-width": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "dev": true, "requires": { @@ -13880,7 +13884,7 @@ }, "string-width": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "dev": true, "requires": { diff --git a/src/core/operations/RegularExpression.mjs b/src/core/operations/RegularExpression.mjs index 03918982..cce65c63 100644 --- a/src/core/operations/RegularExpression.mjs +++ b/src/core/operations/RegularExpression.mjs @@ -228,40 +228,29 @@ function regexList (input, regex, displayTotal, matches, captureGroups) { function regexHighlight (input, regex, displayTotal) { let output = "", title = "", - m, hl = 1, - i = 0, total = 0; - while ((m = regex.exec(input))) { - // Moves pointer when an empty string is matched (prevents infinite loop) - if (m.index === regex.lastIndex) { - regex.lastIndex++; - } + output = input.replace(regex, (match, ...args) => { + args.pop(); // Throw away full string + const offset = args.pop(), + groups = args; - // Add up to match - output += Utils.escapeHtml(input.slice(i, m.index)); - - title = `Offset: ${m.index}\n`; - if (m.length > 1) { + title = `Offset: ${offset}\n`; + if (groups.length) { title += "Groups:\n"; - for (let n = 1; n < m.length; ++n) { - title += `\t${n}: ${m[n]}\n`; + for (let i = 0; i < groups.length; i++) { + title += `\t${i+1}: ${Utils.escapeHtml(groups[i])}\n`; } } - // Add match with highlighting - output += "" + Utils.escapeHtml(m[0]) + ""; - // Switch highlight hl = hl === 1 ? 2 : 1; - i = regex.lastIndex; total++; - } - // Add all after final match - output += Utils.escapeHtml(input.slice(i, input.length)); + return `${Utils.escapeHtml(match)}`; + }); if (displayTotal) output = "Total found: " + total + "\n\n" + output; From 3a6b2875d5883824f1dfd2eca3982b2f1bbd0c61 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 8 Jan 2019 17:51:47 +0000 Subject: [PATCH 12/52] 8.19.6 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index d19abbcc..b8c5b5fe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.19.5", + "version": "8.19.6", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 2274ca30..8dbdc291 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.19.5", + "version": "8.19.6", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From cb9ab7a2c9696545fbd3bb61dcf37dd18b4f2eff Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 8 Jan 2019 18:29:07 +0000 Subject: [PATCH 13/52] Fixed 'Maximise output' button functionality --- src/web/App.mjs | 13 ++++++++++--- src/web/OutputWaiter.mjs | 2 ++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/web/App.mjs b/src/web/App.mjs index 1dab16e6..846803a1 100755 --- a/src/web/App.mjs +++ b/src/web/App.mjs @@ -238,12 +238,18 @@ class App { /** * Sets up the adjustable splitter to allow the user to resize areas of the page. + * + * @param {boolean} [minimise=false] - Set this flag if attempting to minimuse frames to 0 width */ - initialiseSplitter() { + initialiseSplitter(minimise=false) { + if (this.columnSplitter) this.columnSplitter.destroy(); + if (this.ioSplitter) this.ioSplitter.destroy(); + this.columnSplitter = Split(["#operations", "#recipe", "#IO"], { sizes: [20, 30, 50], - minSize: [240, 370, 450], + minSize: minimise ? [0, 0, 0] : [240, 370, 450], gutterSize: 4, + expandToMin: false, onDrag: function() { this.manager.recipe.adjustWidth(); }.bind(this) @@ -251,7 +257,8 @@ class App { this.ioSplitter = Split(["#input", "#output"], { direction: "vertical", - gutterSize: 4 + gutterSize: 4, + minSize: minimise ? [0, 0] : [100, 100] }); this.resetLayout(); diff --git a/src/web/OutputWaiter.mjs b/src/web/OutputWaiter.mjs index 7203a16f..28deaffa 100755 --- a/src/web/OutputWaiter.mjs +++ b/src/web/OutputWaiter.mjs @@ -319,6 +319,7 @@ class OutputWaiter { const el = e.target.id === "maximise-output" ? e.target : e.target.parentNode; if (el.getAttribute("data-original-title").indexOf("Maximise") === 0) { + this.app.initialiseSplitter(true); this.app.columnSplitter.collapse(0); this.app.columnSplitter.collapse(1); this.app.ioSplitter.collapse(0); @@ -328,6 +329,7 @@ class OutputWaiter { } else { $(el).attr("data-original-title", "Maximise output pane"); el.querySelector("i").innerHTML = "fullscreen"; + this.app.initialiseSplitter(false); this.app.resetLayout(); } } From fe1332f18e2ea23a92508ed2fc6d4b1fa42c7b32 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Tue, 8 Jan 2019 18:29:14 +0000 Subject: [PATCH 14/52] 8.19.7 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index b8c5b5fe..81136cc8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.19.6", + "version": "8.19.7", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 8dbdc291..e1b801d2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.19.6", + "version": "8.19.7", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From df8abb099c77babbe6a2a30f4cafc5c3ec6d7207 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 8 Jan 2019 22:23:14 +0000 Subject: [PATCH 15/52] Added code argtype --- package-lock.json | 74 +++++++++++++++---------------- src/core/operations/YaraRules.mjs | 2 +- src/web/HTMLIngredient.mjs | 10 +++++ 3 files changed, 48 insertions(+), 38 deletions(-) diff --git a/package-lock.json b/package-lock.json index eae5e06f..7241c6de 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1571,7 +1571,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -2779,7 +2779,7 @@ }, "string-width": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "dev": true, "requires": { @@ -3322,7 +3322,7 @@ }, "regexpu-core": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/regexpu-core/-/regexpu-core-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-1.0.0.tgz", "integrity": "sha1-hqdj9Y7k18L2sQLkdkBQ3n7ZDGs=", "dev": true, "requires": { @@ -3339,7 +3339,7 @@ }, "regjsparser": { "version": "0.1.5", - "resolved": "http://registry.npmjs.org/regjsparser/-/regjsparser-0.1.5.tgz", + "resolved": "https://registry.npmjs.org/regjsparser/-/regjsparser-0.1.5.tgz", "integrity": "sha1-fuj4Tcb6eS0/0K4ijSS9lJ6tIFw=", "dev": true, "requires": { @@ -4271,7 +4271,7 @@ }, "source-map": { "version": "0.1.43", - "resolved": "http://registry.npmjs.org/source-map/-/source-map-0.1.43.tgz", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.1.43.tgz", "integrity": "sha1-wkvBRspRfBRx9drL4lcbK3+eM0Y=", "requires": { "amdefine": ">=0.0.4" @@ -4461,7 +4461,7 @@ "dependencies": { "source-map": { "version": "0.5.0", - "resolved": "http://registry.npmjs.org/source-map/-/source-map-0.5.0.tgz", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.0.tgz", "integrity": "sha1-D+llA6yGpa213mP05BKuSHLNvoY=", "dev": true } @@ -4829,7 +4829,7 @@ }, "findup-sync": { "version": "0.3.0", - "resolved": "http://registry.npmjs.org/findup-sync/-/findup-sync-0.3.0.tgz", + "resolved": "https://registry.npmjs.org/findup-sync/-/findup-sync-0.3.0.tgz", "integrity": "sha1-N5MKpdgWt3fANEXhlmzGeQpMCxY=", "dev": true, "requires": { @@ -4896,7 +4896,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -5019,7 +5019,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -5660,7 +5660,7 @@ }, "string-width": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "dev": true, "requires": { @@ -5823,7 +5823,7 @@ }, "globby": { "version": "6.1.0", - "resolved": "http://registry.npmjs.org/globby/-/globby-6.1.0.tgz", + "resolved": "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz", "integrity": "sha1-9abXDoOV4hyFj7BInWTfAkJNUGw=", "dev": true, "requires": { @@ -5935,7 +5935,7 @@ }, "resolve": { "version": "1.1.7", - "resolved": "http://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz", "integrity": "sha1-IDEU2CrSxe2ejgQRs5ModeiJ6Xs=", "dev": true } @@ -6409,7 +6409,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -6970,7 +6970,7 @@ }, "is-accessor-descriptor": { "version": "0.1.6", - "resolved": "http://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", "dev": true, "requires": { @@ -7025,7 +7025,7 @@ }, "is-data-descriptor": { "version": "0.1.4", - "resolved": "http://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", "dev": true, "requires": { @@ -7429,7 +7429,7 @@ }, "underscore": { "version": "1.8.3", - "resolved": "http://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz", "integrity": "sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI=", "dev": true } @@ -8227,7 +8227,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -8596,7 +8596,7 @@ }, "multimatch": { "version": "2.1.0", - "resolved": "http://registry.npmjs.org/multimatch/-/multimatch-2.1.0.tgz", + "resolved": "https://registry.npmjs.org/multimatch/-/multimatch-2.1.0.tgz", "integrity": "sha1-nHkGoi+0wCkZ4vX3UWG0zb1LKis=", "dev": true, "requires": { @@ -8608,7 +8608,7 @@ }, "mute-stream": { "version": "0.0.7", - "resolved": "http://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", "dev": true }, @@ -8932,7 +8932,7 @@ }, "underscore": { "version": "1.1.7", - "resolved": "http://registry.npmjs.org/underscore/-/underscore-1.1.7.tgz", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.1.7.tgz", "integrity": "sha1-QLq4S60Z0jAJbo1u9ii/8FXYPbA=" } } @@ -9431,7 +9431,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -9519,7 +9519,7 @@ }, "path-browserify": { "version": "0.0.0", - "resolved": "http://registry.npmjs.org/path-browserify/-/path-browserify-0.0.0.tgz", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-0.0.0.tgz", "integrity": "sha1-oLhwcpquIUAFt9UDLsLLuw+0RRo=", "dev": true }, @@ -10495,7 +10495,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -10788,7 +10788,7 @@ "dependencies": { "underscore": { "version": "1.6.0", - "resolved": "http://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag=", "dev": true } @@ -11107,7 +11107,7 @@ "dependencies": { "source-map": { "version": "0.4.4", - "resolved": "http://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=", "dev": true, "requires": { @@ -11715,7 +11715,7 @@ }, "split2": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/split2/-/split2-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/split2/-/split2-1.1.1.tgz", "integrity": "sha1-Fi2bGIZfAqsvKtlYVSLbm1TEgfk=", "dev": true, "requires": { @@ -11724,7 +11724,7 @@ }, "sprintf-js": { "version": "1.0.3", - "resolved": "http://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", "dev": true }, @@ -11856,7 +11856,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -11873,7 +11873,7 @@ }, "stream-browserify": { "version": "2.0.1", - "resolved": "http://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.1.tgz", + "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.1.tgz", "integrity": "sha1-ZiZu5fm9uZQKTkUUyvtDu3Hlyds=", "dev": true, "requires": { @@ -11959,7 +11959,7 @@ }, "string_decoder": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", "dev": true, "requires": { @@ -12076,7 +12076,7 @@ }, "supports-color": { "version": "2.0.0", - "resolved": "http://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" }, "symbol-tree": { @@ -12539,7 +12539,7 @@ }, "tty-browserify": { "version": "0.0.0", - "resolved": "http://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz", + "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz", "integrity": "sha1-oVe6QC2iTpv5V/mqadUk7tQpAaY=", "dev": true }, @@ -12614,7 +12614,7 @@ }, "underscore": { "version": "1.7.0", - "resolved": "http://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz", "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=" }, "underscore-contrib": { @@ -12628,7 +12628,7 @@ "dependencies": { "underscore": { "version": "1.6.0", - "resolved": "http://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz", "integrity": "sha1-izixDKze9jM3uLJOT/htRa6lKag=", "dev": true } @@ -13013,7 +13013,7 @@ }, "vm-browserify": { "version": "0.0.4", - "resolved": "http://registry.npmjs.org/vm-browserify/-/vm-browserify-0.0.4.tgz", + "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-0.0.4.tgz", "integrity": "sha1-XX6kW7755Kb/ZflUOOCofDV9WnM=", "dev": true, "requires": { @@ -13730,7 +13730,7 @@ }, "string-width": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "dev": true, "requires": { @@ -13885,7 +13885,7 @@ }, "string-width": { "version": "1.0.2", - "resolved": "http://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "dev": true, "requires": { diff --git a/src/core/operations/YaraRules.mjs b/src/core/operations/YaraRules.mjs index 9f2d9b79..3037bdb6 100644 --- a/src/core/operations/YaraRules.mjs +++ b/src/core/operations/YaraRules.mjs @@ -27,7 +27,7 @@ class YaraRules extends Operation { this.outputType = "string"; this.args = [{ name: "Rules", - type: "string", + type: "code", value: "" }]; } diff --git a/src/web/HTMLIngredient.mjs b/src/web/HTMLIngredient.mjs index bb01d7de..0cbb2dad 100755 --- a/src/web/HTMLIngredient.mjs +++ b/src/web/HTMLIngredient.mjs @@ -42,6 +42,16 @@ class HTMLIngredient { i, m; switch (this.type) { + case "code": + html+= `
+ + + ${this.hint ? "" + this.hint + "" : ""}`; + break; case "string": case "binaryString": case "byteArray": From 4c1521a98ee41cd447f691673acbdf08baafa2c3 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 8 Jan 2019 23:26:14 +0000 Subject: [PATCH 16/52] No data matches & warnings support --- src/core/operations/YaraRules.mjs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/core/operations/YaraRules.mjs b/src/core/operations/YaraRules.mjs index 3037bdb6..74de6f29 100644 --- a/src/core/operations/YaraRules.mjs +++ b/src/core/operations/YaraRules.mjs @@ -40,23 +40,32 @@ class YaraRules extends Operation { run(input, args) { return new Promise((resolve, reject) => { Yara().then(yara => { + let matchString = ""; const resp = yara.run(input, args[0]); if (resp.compileErrors.size() > 0) { for (let i = 0; i < resp.compileErrors.size(); i++) { const compileError = resp.compileErrors.get(i); - reject(new OperationError(`Error on line ${compileError.lineNumber}: ${compileError.message}`)); + if (!compileError.warning) { + reject(new OperationError(`Error on line ${compileError.lineNumber}: ${compileError.message}`)); + } else { + matchString += `Warning on line ${compileError.lineNumber}: ${compileError.message}`; + } } } const matchedRules = resp.matchedRules; - let matchString = ""; for (let i = 0; i < matchedRules.keys().size(); i++) { const ruleMatches = matchedRules.get(matchedRules.keys().get(i)); - matchString += `Rule "${matchedRules.keys().get(i)}" matches:\n`; + if (ruleMatches.size() === 0) { + matchString += `Input matches rule "${matchedRules.keys().get(i)}".\n`; + } else { + matchString += `Rule "${matchedRules.keys().get(i)}" matches:\n`; - for (let j = 0; j < ruleMatches.size(); j++) { - const match = ruleMatches.get(j); - matchString += `Position ${match.location}, length ${match.matchLength}, data: ${match.data}\n`; + for (let j = 0; j < ruleMatches.size(); j++) { + const match = ruleMatches.get(j); + matchString += `Position ${match.location}, length ${match.matchLength}, data: ${match.data}\n`; + } } + } resolve(matchString); }); From 26a2fb66625cb9bd703ef49446f7aeca6436028e Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 9 Jan 2019 09:56:55 +0000 Subject: [PATCH 17/52] Increased size of rule inp & expanded memory for wasm --- package-lock.json | 6 +++--- package.json | 2 +- src/web/HTMLIngredient.mjs | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 7241c6de..8827fd47 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7756,9 +7756,9 @@ "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA=" }, "libyara-wasm": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.2.tgz", - "integrity": "sha512-Q4Biyfwiwrz7u25j7HNbGjJkcuekMJJeGCwbUgc7s7GMriZBSctCJPzlP5MBKgMzGMe0fNc9Hu5EP6Y1lBFnWg==" + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.4.tgz", + "integrity": "sha512-Puw8AfHRgAiS2SBvJBlh3DEYU3icU16MciwQK5Fsxel021UK7DcY1A5DAKYanPNeXVztlz/9USZbEneAkcWzvA==" }, "livereload-js": { "version": "2.4.0", diff --git a/package.json b/package.json index 1f207e4b..ffdc5dfd 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,7 @@ "jsqr": "^1.1.1", "jsrsasign": "8.0.12", "kbpgp": "^2.0.82", - "libyara-wasm": "0.0.2", + "libyara-wasm": "0.0.4", "lodash": "^4.17.11", "loglevel": "^1.6.1", "loglevel-message-prefix": "^3.0.0", diff --git a/src/web/HTMLIngredient.mjs b/src/web/HTMLIngredient.mjs index 0cbb2dad..4de7e43f 100755 --- a/src/web/HTMLIngredient.mjs +++ b/src/web/HTMLIngredient.mjs @@ -49,6 +49,7 @@ class HTMLIngredient { id="${this.id}" arg-name="${this.name}" value="${this.value}" + rows=5 ${this.disabled ? "disabled" : ""}> ${this.hint ? "" + this.hint + "" : ""}`; break; From 4db2335107ddf519784b4033336087ebf16ec2b0 Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 9 Jan 2019 11:45:11 +0000 Subject: [PATCH 18/52] Speedrunning strats (increased speed on big files) --- package-lock.json | 6 +++--- package.json | 2 +- src/core/operations/YaraRules.mjs | 9 +++++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8827fd47..573f8e67 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7756,9 +7756,9 @@ "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA=" }, "libyara-wasm": { - "version": "0.0.4", - "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.4.tgz", - "integrity": "sha512-Puw8AfHRgAiS2SBvJBlh3DEYU3icU16MciwQK5Fsxel021UK7DcY1A5DAKYanPNeXVztlz/9USZbEneAkcWzvA==" + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.6.tgz", + "integrity": "sha512-Crnaz5G/ejjZrEYTlyUZIaquR66djW8w8UR8GtgFrpWzhiySPJTcdxwOhGmCku2VhhETPznz20KxBNifBSF+oA==" }, "livereload-js": { "version": "2.4.0", diff --git a/package.json b/package.json index ffdc5dfd..2f9c7d04 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,7 @@ "jsqr": "^1.1.1", "jsrsasign": "8.0.12", "kbpgp": "^2.0.82", - "libyara-wasm": "0.0.4", + "libyara-wasm": "0.0.6", "lodash": "^4.17.11", "loglevel": "^1.6.1", "loglevel-message-prefix": "^3.0.0", diff --git a/src/core/operations/YaraRules.mjs b/src/core/operations/YaraRules.mjs index 74de6f29..b6d78a8c 100644 --- a/src/core/operations/YaraRules.mjs +++ b/src/core/operations/YaraRules.mjs @@ -23,7 +23,7 @@ class YaraRules extends Operation { this.module = "Yara"; this.description = "Yara support"; this.infoURL = "https://en.wikipedia.org/wiki/YARA"; - this.inputType = "string"; + this.inputType = "ArrayBuffer"; this.outputType = "string"; this.args = [{ name: "Rules", @@ -41,7 +41,12 @@ class YaraRules extends Operation { return new Promise((resolve, reject) => { Yara().then(yara => { let matchString = ""; - const resp = yara.run(input, args[0]); + const inpArr = new Uint8Array(input); // I know this is garbage but it's like 1.5 times faster + const inpVec = new yara.vectorChar(); + for (let i = 0; i < inpArr.length; i++) { + inpVec.push_back(inpArr[i]); + } + const resp = yara.run(inpVec, args[0]); if (resp.compileErrors.size() > 0) { for (let i = 0; i < resp.compileErrors.size(); i++) { const compileError = resp.compileErrors.get(i); From ebb632e8882f8f2bad67429c4bf49ab13a211587 Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 9 Jan 2019 14:29:14 +0000 Subject: [PATCH 19/52] Added metadata, string identifiers and operation args --- package-lock.json | 6 ++-- package.json | 2 +- src/core/operations/YaraRules.mjs | 58 +++++++++++++++++++++++-------- src/web/HTMLIngredient.mjs | 3 +- 4 files changed, 49 insertions(+), 20 deletions(-) diff --git a/package-lock.json b/package-lock.json index 573f8e67..2b4d058b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7756,9 +7756,9 @@ "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA=" }, "libyara-wasm": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.6.tgz", - "integrity": "sha512-Crnaz5G/ejjZrEYTlyUZIaquR66djW8w8UR8GtgFrpWzhiySPJTcdxwOhGmCku2VhhETPznz20KxBNifBSF+oA==" + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.8.tgz", + "integrity": "sha512-ZB+Ya3bEBoanvde47X8RzqpMBHgrPxrTZIJ/UEoatVnOEy2he1IORuotdSkP2o73URRzHGN1jNWDIhTdfbZ3rQ==" }, "livereload-js": { "version": "2.4.0", diff --git a/package.json b/package.json index 2f9c7d04..8e21a0a6 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,7 @@ "jsqr": "^1.1.1", "jsrsasign": "8.0.12", "kbpgp": "^2.0.82", - "libyara-wasm": "0.0.6", + "libyara-wasm": "0.0.8", "lodash": "^4.17.11", "loglevel": "^1.6.1", "loglevel-message-prefix": "^3.0.0", diff --git a/src/core/operations/YaraRules.mjs b/src/core/operations/YaraRules.mjs index b6d78a8c..bfdebb13 100644 --- a/src/core/operations/YaraRules.mjs +++ b/src/core/operations/YaraRules.mjs @@ -25,11 +25,28 @@ class YaraRules extends Operation { this.infoURL = "https://en.wikipedia.org/wiki/YARA"; this.inputType = "ArrayBuffer"; this.outputType = "string"; - this.args = [{ - name: "Rules", - type: "code", - value: "" - }]; + this.args = [ + { + name: "Rules", + type: "code", + value: "" + }, + { + name: "Show strings", + type: "boolean", + value: false + }, + { + name: "Show string lengths", + type: "boolean", + value: false + }, + { + name: "Show metadata", + type: "boolean", + value: false + } + ]; } /** @@ -38,6 +55,7 @@ class YaraRules extends Operation { * @returns {string} */ run(input, args) { + const [rules, showStrings, showLengths, showMeta] = args; return new Promise((resolve, reject) => { Yara().then(yara => { let matchString = ""; @@ -46,7 +64,7 @@ class YaraRules extends Operation { for (let i = 0; i < inpArr.length; i++) { inpVec.push_back(inpArr[i]); } - const resp = yara.run(inpVec, args[0]); + const resp = yara.run(inpVec, rules); if (resp.compileErrors.size() > 0) { for (let i = 0; i < resp.compileErrors.size(); i++) { const compileError = resp.compileErrors.get(i); @@ -58,16 +76,26 @@ class YaraRules extends Operation { } } const matchedRules = resp.matchedRules; - for (let i = 0; i < matchedRules.keys().size(); i++) { - const ruleMatches = matchedRules.get(matchedRules.keys().get(i)); - if (ruleMatches.size() === 0) { - matchString += `Input matches rule "${matchedRules.keys().get(i)}".\n`; + for (let i = 0; i < matchedRules.size(); i++) { + const rule = matchedRules.get(i); + const matches = rule.resolvedMatches; + let meta = ""; + if (showMeta && rule.metadata.size() > 0) { + meta += " ["; + for (let j = 0; j < rule.metadata.size(); j++) { + meta += `${rule.metadata.get(j).identifier}: ${rule.metadata.get(j).data}, `; + } + meta = meta.slice(0, -2) + "]"; + } + if (matches.size() === 0 || !(showStrings || showLengths)) { + matchString += `Input matches rule "${rule.ruleName}"${meta}.\n`; } else { - matchString += `Rule "${matchedRules.keys().get(i)}" matches:\n`; - - for (let j = 0; j < ruleMatches.size(); j++) { - const match = ruleMatches.get(j); - matchString += `Position ${match.location}, length ${match.matchLength}, data: ${match.data}\n`; + matchString += `Rule "${rule.ruleName}"${meta} matches:\n`; + for (let j = 0; j < matches.size(); j++) { + const match = matches.get(j); + if (showStrings || showLengths) { + matchString += `Pos ${match.location}, ${showLengths ? `length ${match.matchLength}, ` : ""}identifier ${match.stringIdentifier}${showStrings ? `, data: "${match.data}"` : ""}\n`; + } } } diff --git a/src/web/HTMLIngredient.mjs b/src/web/HTMLIngredient.mjs index 4de7e43f..234c5343 100755 --- a/src/web/HTMLIngredient.mjs +++ b/src/web/HTMLIngredient.mjs @@ -51,7 +51,8 @@ class HTMLIngredient { value="${this.value}" rows=5 ${this.disabled ? "disabled" : ""}> - ${this.hint ? "" + this.hint + "" : ""}`; + ${this.hint ? "" + this.hint + "" : ""} +
`; break; case "string": case "binaryString": From dd9ba4d250b381e45966b74e9109e95b2306939b Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 9 Jan 2019 15:28:50 +0000 Subject: [PATCH 20/52] Fixed problems flagged by n's review --- src/core/Ingredient.mjs | 2 ++ src/core/Operation.mjs | 1 + .../{YaraRules.mjs => YARARules.mjs} | 24 +++++++++++-------- src/web/HTMLIngredient.mjs | 14 ++--------- 4 files changed, 19 insertions(+), 22 deletions(-) rename src/core/operations/{YaraRules.mjs => YARARules.mjs} (79%) diff --git a/src/core/Ingredient.mjs b/src/core/Ingredient.mjs index 00dd5f6d..96cdd400 100755 --- a/src/core/Ingredient.mjs +++ b/src/core/Ingredient.mjs @@ -23,6 +23,7 @@ class Ingredient { this._value = null; this.disabled = false; this.hint = ""; + this.rows = 0; this.toggleValues = []; this.target = null; this.defaultIndex = 0; @@ -45,6 +46,7 @@ class Ingredient { this.defaultValue = ingredientConfig.value; this.disabled = !!ingredientConfig.disabled; this.hint = ingredientConfig.hint || false; + this.rows = ingredientConfig.rows || false; this.toggleValues = ingredientConfig.toggleValues; this.target = typeof ingredientConfig.target !== "undefined" ? ingredientConfig.target : null; this.defaultIndex = typeof ingredientConfig.defaultIndex !== "undefined" ? ingredientConfig.defaultIndex : 0; diff --git a/src/core/Operation.mjs b/src/core/Operation.mjs index 3f6b3e86..d57f885d 100755 --- a/src/core/Operation.mjs +++ b/src/core/Operation.mjs @@ -179,6 +179,7 @@ class Operation { if (ing.toggleValues) conf.toggleValues = ing.toggleValues; if (ing.hint) conf.hint = ing.hint; + if (ing.rows) conf.rows = ing.rows; if (ing.disabled) conf.disabled = ing.disabled; if (ing.target) conf.target = ing.target; if (ing.defaultIndex) conf.defaultIndex = ing.defaultIndex; diff --git a/src/core/operations/YaraRules.mjs b/src/core/operations/YARARules.mjs similarity index 79% rename from src/core/operations/YaraRules.mjs rename to src/core/operations/YARARules.mjs index bfdebb13..094d9f43 100644 --- a/src/core/operations/YaraRules.mjs +++ b/src/core/operations/YARARules.mjs @@ -9,41 +9,45 @@ import OperationError from "../errors/OperationError"; import Yara from "libyara-wasm"; /** - * Yara Rules operation + * YARA Rules operation */ -class YaraRules extends Operation { +class YARARules extends Operation { /** - * YaraRules constructor + * YARARules constructor */ constructor() { super(); - this.name = "Yara Rules"; + this.name = "YARA Rules"; this.module = "Yara"; - this.description = "Yara support"; - this.infoURL = "https://en.wikipedia.org/wiki/YARA"; + this.description = "YARA is a tool developed at VirusTotal, primarily aimed at helping malware researchers to identify and classify malware samples. It matches based on rules specified by the user containing textual or binary patterns and a boolean expression. For help on writing rules, see the YARA documentation."; + this.infoURL = "https://wikipedia.org/wiki/YARA"; this.inputType = "ArrayBuffer"; this.outputType = "string"; this.args = [ { name: "Rules", - type: "code", - value: "" + type: "text", + value: "", + rows: 5 }, { name: "Show strings", type: "boolean", + hint: "Show each match's data", value: false }, { name: "Show string lengths", type: "boolean", + hint: "Show the length of each match's data", value: false }, { name: "Show metadata", type: "boolean", + hint: "Show the metadata of each rule", value: false } ]; @@ -59,7 +63,7 @@ class YaraRules extends Operation { return new Promise((resolve, reject) => { Yara().then(yara => { let matchString = ""; - const inpArr = new Uint8Array(input); // I know this is garbage but it's like 1.5 times faster + const inpArr = new Uint8Array(input); const inpVec = new yara.vectorChar(); for (let i = 0; i < inpArr.length; i++) { inpVec.push_back(inpArr[i]); @@ -107,4 +111,4 @@ class YaraRules extends Operation { } -export default YaraRules; +export default YARARules; diff --git a/src/web/HTMLIngredient.mjs b/src/web/HTMLIngredient.mjs index 234c5343..59b7bec7 100755 --- a/src/web/HTMLIngredient.mjs +++ b/src/web/HTMLIngredient.mjs @@ -25,6 +25,7 @@ class HTMLIngredient { this.value = config.value; this.disabled = config.disabled || false; this.hint = config.hint || false; + this.rows = config.rows || false; this.target = config.target; this.defaultIndex = config.defaultIndex || 0; this.toggleValues = config.toggleValues; @@ -42,18 +43,6 @@ class HTMLIngredient { i, m; switch (this.type) { - case "code": - html+= `
- - - ${this.hint ? "" + this.hint + "" : ""} -
`; - break; case "string": case "binaryString": case "byteArray": @@ -241,6 +230,7 @@ class HTMLIngredient { class="form-control arg" id="${this.id}" arg-name="${this.name}" + rows="${this.rows ? this.rows : 3}" ${this.disabled ? "disabled" : ""}>${this.value} ${this.hint ? "" + this.hint + "" : ""} `; From c49a770c59adb28cfd082dc2c935f7c900902625 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 9 Jan 2019 16:36:34 +0000 Subject: [PATCH 21/52] Tidied up Lorem Ipsum op --- CHANGELOG.md | 5 ++ src/core/config/Categories.json | 4 +- src/core/lib/LoremIpsum.mjs | 81 ++++++++++--------- ...umGenerator.mjs => GenerateLoremIpsum.mjs} | 12 +-- 4 files changed, 58 insertions(+), 44 deletions(-) rename src/core/operations/{LoremIpsumGenerator.mjs => GenerateLoremIpsum.mjs} (85%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a71bb41..d9b8eff0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master). +### [8.20.0] - 2019-01-09 +- 'Generate Lorem Ipsum' operation added [@klaxon1] | [#455] + ### [8.19.0] - 2018-12-30 - UI test suite added to confirm that the app loads correctly in a reasonable time and that various operations from each module can be run [@n1474335] | [#458] @@ -88,6 +91,7 @@ All major and minor version changes will be documented in this file. Details of +[8.20.0]: https://github.com/gchq/CyberChef/releases/tag/v8.20.0 [8.19.0]: https://github.com/gchq/CyberChef/releases/tag/v8.19.0 [8.18.0]: https://github.com/gchq/CyberChef/releases/tag/v8.18.0 [8.17.0]: https://github.com/gchq/CyberChef/releases/tag/v8.17.0 @@ -159,4 +163,5 @@ All major and minor version changes will be documented in this file. Details of [#446]: https://github.com/gchq/CyberChef/pull/446 [#448]: https://github.com/gchq/CyberChef/pull/448 [#449]: https://github.com/gchq/CyberChef/pull/449 +[#455]: https://github.com/gchq/CyberChef/pull/455 [#458]: https://github.com/gchq/CyberChef/pull/458 diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 3678a88b..3e792b99 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -374,9 +374,9 @@ "Generate QR Code", "Parse QR Code", "Haversine distance", + "Generate Lorem Ipsum", "Numberwang", - "XKCD Random Number", - "Lorem Ipsum Generator" + "XKCD Random Number" ] }, { diff --git a/src/core/lib/LoremIpsum.mjs b/src/core/lib/LoremIpsum.mjs index 9712a429..d7fff69b 100644 --- a/src/core/lib/LoremIpsum.mjs +++ b/src/core/lib/LoremIpsum.mjs @@ -2,16 +2,16 @@ * Lorem Ipsum generator. * * @author Klaxon [klaxon@veyr.com] - * @copyright Crown Copyright 2016 + * @copyright Crown Copyright 2018 * @license Apache-2.0 */ - /** - * generate lorem ipsum paragraphs. - * - * @param {number} length - * @returns {string} - */ +/** + * Generate lorem ipsum paragraphs. + * + * @param {number} length + * @returns {string} + */ export function GenerateParagraphs(length=3) { const paragraphs = []; while (paragraphs.length < length) { @@ -29,12 +29,13 @@ export function GenerateParagraphs(length=3) { return paragraphs.join(""); } + /** -* generate lorem ipsum sentences. -* -* @param {number} length -* @returns {string} -*/ + * Generate lorem ipsum sentences. + * + * @param {number} length + * @returns {string} + */ export function GenerateSentences(length=3) { const sentences = []; while (sentences.length < length) { @@ -46,12 +47,13 @@ export function GenerateSentences(length=3) { return paragraphs.join(""); } + /** -* generate lorem ipsum words. -* -* @param {number} length -* @returns {string} -*/ + * Generate lorem ipsum words. + * + * @param {number} length + * @returns {string} + */ export function GenerateWords(length=3) { const words = getWords(length); const sentences = wordsToSentences(words); @@ -59,19 +61,21 @@ export function GenerateWords(length=3) { return paragraphs.join(""); } - /** - * generate lorem ipsum bytes. - * - * @param {number} length - * @returns {string} - */ + +/** + * Generate lorem ipsum bytes. + * + * @param {number} length + * @returns {string} + */ export function GenerateBytes(length=3) { const str = GenerateWords(length/3); return str.slice(0, length); } + /** - * get array of randomly selected words from the lorem ipsum wordList. + * Get array of randomly selected words from the lorem ipsum wordList. * * @param {number} length * @returns {string[]} @@ -84,16 +88,16 @@ function getWords(length=3) { while (words.length < length){ do { word = wordList[Math.floor(Math.random() * wordList.length)]; - } - while (previousWord === word); + } while (previousWord === word); words.push(word); previousWord = word; } return words; } + /** - * convert an array or words into an array of sentences" + * Convert an array of words into an array of sentences * * @param {string[]} words * @returns {string[]} @@ -112,8 +116,9 @@ function wordsToSentences(words) { return sentences; } + /** - * convert an array or sentences into an array of paragraphs" + * Convert an array of sentences into an array of paragraphs * * @param {string[]} sentences * @returns {string[]} @@ -130,15 +135,16 @@ function sentencesToParagraphs(sentences) { return paragraphs; } + /** - * format an array of words into a sentence. + * Format an array of words into a sentence. * * @param {string[]} words * @returns {string} * @private */ function formatSentence(words) { - //0.35 chance of a comma being added randomly to the sentence. + // 0.35 chance of a comma being added randomly to the sentence. if (Math.random() < PROBABILITY_OF_A_COMMA) { const pos = Math.round(Math.random()*(words.length-1)); words[pos] +=","; @@ -149,8 +155,9 @@ function formatSentence(words) { return sentence; } + /** - * format an array of sentences into a paragraph + * Format an array of sentences into a paragraph. * * @param {string[]} sentences * @returns {string} @@ -162,10 +169,11 @@ function formatParagraph(sentences) { return paragraph; } + /** - * get a random number based on a mean and standard deviation. + * Get a random number based on a mean and standard deviation. * - * @param {number} Mean + * @param {number} mean * @param {number} stdDev * @returns {number} * @private @@ -174,13 +182,13 @@ function getRandomLength(mean, stdDev) { let length; do { length = Math.round((Math.random()*2-1)+(Math.random()*2-1)+(Math.random()*2-1)*stdDev+mean); - } - while (length <= 0); + } while (length <= 0); return length; } + /** - * replace first 5 words with "Lorem ipsum dolor sit amet" + * Replace first 5 words with "Lorem ipsum dolor sit amet" * * @param {string[]} str * @returns {string[]} @@ -200,6 +208,7 @@ function replaceStart(str) { } } + const SENTENCE_LENGTH_MEAN = 15; const SENTENCE_LENGTH_STD_DEV = 9; const PARAGRAPH_LENGTH_MEAN = 5; diff --git a/src/core/operations/LoremIpsumGenerator.mjs b/src/core/operations/GenerateLoremIpsum.mjs similarity index 85% rename from src/core/operations/LoremIpsumGenerator.mjs rename to src/core/operations/GenerateLoremIpsum.mjs index 228daaa1..fb5ecd17 100644 --- a/src/core/operations/LoremIpsumGenerator.mjs +++ b/src/core/operations/GenerateLoremIpsum.mjs @@ -9,17 +9,17 @@ import OperationError from "../errors/OperationError"; import { GenerateParagraphs, GenerateSentences, GenerateWords, GenerateBytes } from "../lib/LoremIpsum"; /** - * Lorem Ipsum Generator operation + * Generate Lorem Ipsum operation */ -class LoremIpsumGenerator extends Operation { +class GenerateLoremIpsum extends Operation { /** - * LoremIpsumGenerator constructor + * GenerateLoremIpsum constructor */ constructor() { super(); - this.name = "Lorem Ipsum Generator"; + this.name = "Generate Lorem Ipsum"; this.module = "Default"; this.description = "Generate varying length lorem ipsum placeholder text."; this.infoURL = "https://wikipedia.org/wiki/Lorem_ipsum"; @@ -60,11 +60,11 @@ class LoremIpsumGenerator extends Operation { case "Bytes": return GenerateBytes(length); default: - throw new OperationError("invalid lengthType"); + throw new OperationError("Invalid length type"); } } } -export default LoremIpsumGenerator; +export default GenerateLoremIpsum; From 324c409ff11c563fa8d4dce0ccef6460d4934d1d Mon Sep 17 00:00:00 2001 From: n1474335 Date: Wed, 9 Jan 2019 16:38:53 +0000 Subject: [PATCH 22/52] 8.20.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 81136cc8..fb28121d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.19.7", + "version": "8.20.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index e1b801d2..bdc53f20 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.19.7", + "version": "8.20.0", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From 995fcab071e187c092ee085bc6e5c41c817e9590 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 10 Jan 2019 15:01:01 +0000 Subject: [PATCH 23/52] Tidied up Case Insensitive Regex ops --- CHANGELOG.md | 6 ++++++ src/core/operations/FromCaseInsensitiveRegex.mjs | 2 +- src/core/operations/ToCaseInsensitiveRegex.mjs | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9b8eff0..4761b540 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master). +### [8.21.0] - 2019-01-10 +- 'To Case Insensitive Regex' and 'From Case Insensitive Regex' operations added [@masq] | [#461] + ### [8.20.0] - 2019-01-09 - 'Generate Lorem Ipsum' operation added [@klaxon1] | [#455] @@ -91,6 +94,7 @@ All major and minor version changes will be documented in this file. Details of +[8.21.0]: https://github.com/gchq/CyberChef/releases/tag/v8.21.0 [8.20.0]: https://github.com/gchq/CyberChef/releases/tag/v8.20.0 [8.19.0]: https://github.com/gchq/CyberChef/releases/tag/v8.19.0 [8.18.0]: https://github.com/gchq/CyberChef/releases/tag/v8.18.0 @@ -134,6 +138,7 @@ All major and minor version changes will be documented in this file. Details of [@tcode2k16]: https://github.com/tcode2k16 [@Cynser]: https://github.com/Cynser [@anthony-arnold]: https://github.com/anthony-arnold +[@masq]: https://github.com/masq [#95]: https://github.com/gchq/CyberChef/pull/299 [#173]: https://github.com/gchq/CyberChef/pull/173 @@ -165,3 +170,4 @@ All major and minor version changes will be documented in this file. Details of [#449]: https://github.com/gchq/CyberChef/pull/449 [#455]: https://github.com/gchq/CyberChef/pull/455 [#458]: https://github.com/gchq/CyberChef/pull/458 +[#461]: https://github.com/gchq/CyberChef/pull/461 diff --git a/src/core/operations/FromCaseInsensitiveRegex.mjs b/src/core/operations/FromCaseInsensitiveRegex.mjs index 2448c5e5..36cd9b14 100644 --- a/src/core/operations/FromCaseInsensitiveRegex.mjs +++ b/src/core/operations/FromCaseInsensitiveRegex.mjs @@ -19,7 +19,7 @@ class FromCaseInsensitiveRegex extends Operation { this.name = "From Case Insensitive Regex"; this.module = "Default"; - this.description = "Converts a case-insensitive regex string to a case sensitive regex string (no guarantee on it being the proper original casing) in case /i wasn't available at the time but now is, or you need it to be case-sensitive again."; + this.description = "Converts a case-insensitive regex string to a case sensitive regex string (no guarantee on it being the proper original casing) in case the i flag wasn't available at the time but now is, or you need it to be case-sensitive again.

e.g. [mM][oO][zZ][iI][lL][lL][aA]/[0-9].[0-9] .* becomes Mozilla/[0-9].[0-9] .*"; this.infoURL = "https://wikipedia.org/wiki/Regular_expression"; this.inputType = "string"; this.outputType = "string"; diff --git a/src/core/operations/ToCaseInsensitiveRegex.mjs b/src/core/operations/ToCaseInsensitiveRegex.mjs index 32fb96c7..0d606a05 100644 --- a/src/core/operations/ToCaseInsensitiveRegex.mjs +++ b/src/core/operations/ToCaseInsensitiveRegex.mjs @@ -19,7 +19,7 @@ class ToCaseInsensitiveRegex extends Operation { this.name = "To Case Insensitive Regex"; this.module = "Default"; - this.description = "Converts a case-sensitive regex string into a case-insensitive regex string in case /i flag is unavailable to you."; + this.description = "Converts a case-sensitive regex string into a case-insensitive regex string in case the i flag is unavailable to you.

e.g. Mozilla/[0-9].[0-9] .* becomes [mM][oO][zZ][iI][lL][lL][aA]/[0-9].[0-9] .*"; this.infoURL = "https://wikipedia.org/wiki/Regular_expression"; this.inputType = "string"; this.outputType = "string"; From 863a5256255367889452fd33da3842e987714c09 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 10 Jan 2019 15:02:26 +0000 Subject: [PATCH 24/52] 8.21.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index fb28121d..9078d0c3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.20.0", + "version": "8.21.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index bdc53f20..c0152ab0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.20.0", + "version": "8.21.0", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From 9e63e40dab12874585a82445e3abf63dac6308b1 Mon Sep 17 00:00:00 2001 From: j433866 Date: Thu, 10 Jan 2019 15:24:29 +0000 Subject: [PATCH 25/52] Add new MGRS module and update webpack-dev-server --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 48751e21..92fda38b 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ "url-loader": "^1.1.2", "web-resource-inliner": "^4.2.1", "webpack": "^4.25.1", - "webpack-dev-server": "^3.1.10", + "webpack-dev-server": "^3.1.14", "webpack-node-externals": "^1.7.2", "worker-loader": "^2.0.0" }, @@ -104,6 +104,7 @@ "lodash": "^4.17.11", "loglevel": "^1.6.1", "loglevel-message-prefix": "^3.0.0", + "mgrs": "^1.0.0", "moment": "^2.22.2", "moment-timezone": "^0.5.23", "ngeohash": "^0.6.0", From c2068b343b3ae7c767d341f6c798f0fda35f3c59 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 10 Jan 2019 15:42:48 +0000 Subject: [PATCH 26/52] Tidied up and added global matching to Subsection operation --- CHANGELOG.md | 5 +++++ src/core/operations/Subsection.mjs | 30 +++++++++++++++++++++--------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4761b540..71949c82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master). +### [8.22.0] - 2019-01-10 +- 'Subsection' operation added [@j433866] | [#467] + ### [8.21.0] - 2019-01-10 - 'To Case Insensitive Regex' and 'From Case Insensitive Regex' operations added [@masq] | [#461] @@ -94,6 +97,7 @@ All major and minor version changes will be documented in this file. Details of +[8.22.0]: https://github.com/gchq/CyberChef/releases/tag/v8.22.0 [8.21.0]: https://github.com/gchq/CyberChef/releases/tag/v8.21.0 [8.20.0]: https://github.com/gchq/CyberChef/releases/tag/v8.20.0 [8.19.0]: https://github.com/gchq/CyberChef/releases/tag/v8.19.0 @@ -171,3 +175,4 @@ All major and minor version changes will be documented in this file. Details of [#455]: https://github.com/gchq/CyberChef/pull/455 [#458]: https://github.com/gchq/CyberChef/pull/458 [#461]: https://github.com/gchq/CyberChef/pull/461 +[#467]: https://github.com/gchq/CyberChef/pull/467 diff --git a/src/core/operations/Subsection.mjs b/src/core/operations/Subsection.mjs index 94258b6d..8133d31c 100644 --- a/src/core/operations/Subsection.mjs +++ b/src/core/operations/Subsection.mjs @@ -38,6 +38,11 @@ class Subsection extends Operation { "type": "boolean", "value": true }, + { + "name": "Global matching", + "type": "boolean", + "value": true + }, { "name": "Ignore errors", "type": "boolean", @@ -49,7 +54,7 @@ class Subsection extends Operation { /** * @param {Object} state - The current state of the recipe. * @param {number} state.progress - The current position in the recipe. - * @param {Dish} state.Dish - The Dish being operated on + * @param {Dish} state.dish - The Dish being operated on * @param {Operation[]} state.opList - The list of operations in the recipe * @returns {Object} - The updated state of the recipe */ @@ -59,12 +64,12 @@ class Subsection extends Operation { outputType = opList[state.progress].outputType, input = await state.dish.get(inputType), ings = opList[state.progress].ingValues, - [section, caseSensitive, ignoreErrors] = ings, + [section, caseSensitive, global, ignoreErrors] = ings, subOpList = []; if (input && section !== "") { // Create subOpList for each tranche to operate on - // (all remaining operations unless we encounter a Merge) + // all remaining operations unless we encounter a Merge for (let i = state.progress + 1; i < opList.length; i++) { if (opList[i].name === "Merge" && !opList[i].disabled) { break; @@ -73,13 +78,15 @@ class Subsection extends Operation { } } - let flags = "g", + let flags = "", inOffset = 0, output = "", m, progress = 0; - if (!caseSensitive) - flags += "i"; + + if (!caseSensitive) flags += "i"; + if (global) flags += "g"; + const regex = new XRegExp(section, flags), recipe = new Recipe(); @@ -95,16 +102,19 @@ class Subsection extends Operation { matched = true; // Add up to match let matchStr = m[0]; - if (m.length === 1) { + + if (m.length === 1) { // No capture groups output += input.slice(inOffset, m.index); - inOffset = regex.lastIndex; + inOffset = m.index + m[0].length; } else if (m.length >= 2) { matchStr = m[1]; + // Need to add some of the matched string that isn't in the capture group output += input.slice(inOffset, m.index + m[0].indexOf(m[1])); // Set i to be after the end of the first capture group - inOffset = regex.lastIndex - (m[0].length - (m[0].indexOf(m[1]) + m[1].length)); + inOffset = m.index + m[0].indexOf(m[1]) + m[1].length; } + // Baseline ing values for each tranche so that registers are reset subOpList.forEach((op, i) => { op.ingValues = JSON.parse(JSON.stringify(ingValues[i])); @@ -122,7 +132,9 @@ class Subsection extends Operation { progress = err.progress + 1; } output += await dish.get(outputType); + if (!regex.global) break; } + // If no matches were found, advance progress to after a Merge op // Otherwise, the operations below Subsection will be run on all the input data if (!matched) { From 9787ab04cdc671b9cf1ea8f64306f136eaa9371a Mon Sep 17 00:00:00 2001 From: n1474335 Date: Thu, 10 Jan 2019 15:44:02 +0000 Subject: [PATCH 27/52] 8.22.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9078d0c3..00e57f64 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.21.0", + "version": "8.22.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index c0152ab0..96053213 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.21.0", + "version": "8.22.0", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From abdd70c6fa5f15dd53a6dfff9e5d379ef421d49e Mon Sep 17 00:00:00 2001 From: j433866 Date: Fri, 11 Jan 2019 11:59:13 +0000 Subject: [PATCH 28/52] Add ConvertCoordinates to lib folder --- src/core/lib/ConvertCoordinates.mjs | 210 ++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 src/core/lib/ConvertCoordinates.mjs diff --git a/src/core/lib/ConvertCoordinates.mjs b/src/core/lib/ConvertCoordinates.mjs new file mode 100644 index 00000000..45ab0690 --- /dev/null +++ b/src/core/lib/ConvertCoordinates.mjs @@ -0,0 +1,210 @@ +/** + * Co-ordinate conversion resources. + * + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import geohash from "ngeohash"; +import mgrs from "mgrs"; + +/** + * Co-ordinate formats + */ +export const FORMATS = [ + "Degrees Minutes Seconds", + "Degrees Decimal Minutes", + "Decimal Degrees", + "Geohash", + "Military Grid Reference System" +]; + +/** + * Convert a given latitude and longitude into a different format. + * @param {string} inLat - Input latitude to be converted. Use this for supplying single values for conversion (e.g. geohash) + * @param {string} inLong - Input longitude to be converted + * @param {string} inFormat - Format of the input coordinates + * @param {string} outFormat - Format to convert to + * @param {number} precision - Precision of the result + * @returns {string[]} Array containing the converted latitude and longitude + */ +export function convertCoordinates (inLat, inLong, inFormat, outFormat, precision) { + let convLat = inLat; + let convLong = inLong; + if (inFormat === "Geohash") { + const hash = geohash.decode(inLat); + convLat = hash.latitude.toString(); + convLong = hash.longitude.toString(); + } else if (inFormat === "Military Grid Reference System") { + const result = mgrs.toPoint(inLat.replace(" ", "")); + convLat = result[1]; + convLong = result[0]; + } else { + convLat = convertSingleCoordinate(inLat, inFormat, "Decimal Degrees", 15).split("°"); + convLong = convertSingleCoordinate(inLong, inFormat, "Decimal Degrees", 15).split("°"); + } + + if (outFormat === "Geohash") { + convLat = geohash.encode(parseFloat(convLat), parseFloat(convLong), precision); + } else if (outFormat === "Military Grid Reference System") { + convLat = mgrs.forward([parseFloat(convLong), parseFloat(convLat)], precision); + } else { + convLat = convertSingleCoordinate(convLat.toString(), "Decimal Degrees", outFormat, precision); + convLong = convertSingleCoordinate(convLong.toString(), "Decimal Degrees", outFormat, precision); + } + + return [convLat, convLong]; +} + +/** + * @param {string} input - The input co-ordinate to be converted + * @param {string} inFormat - The format of the input co-ordinates + * @param {string} outFormat - The format which input should be converted to + * @param {boolean} returnRaw - When true, returns the raw float instead of a String + * @returns {string|{Object}} The converted co-ordinate result, as either the raw object or a formatted string + */ +export function convertSingleCoordinate (input, inFormat, outFormat, precision, returnRaw = false){ + let converted; + precision = Math.pow(10, precision); + const convData = splitInput(input); + // Convert everything to decimal degrees first + switch (inFormat) { + case "Degrees Minutes Seconds": + if (convData.length < 3) { + throw "Invalid co-ordinates format."; + } + converted = convDMSToDD(convData[0], convData[1], convData[2], precision); + break; + case "Degrees Decimal Minutes": + if (convData.length < 2) { + throw "Invalid co-ordinates format."; + } + converted = convDDMToDD(convData[0], convData[1], precision); + break; + case "Decimal Degrees": + if (convData.length < 1) { + throw "Invalid co-ordinates format."; + } + converted = convDDToDD(convData[0], precision); + break; + default: + throw "Unknown input format selection."; + } + + // Convert from decimal degrees to the output format + switch (outFormat) { + case "Decimal Degrees": + break; + case "Degrees Minutes Seconds": + converted = convDDToDMS(converted.degrees); + break; + case "Degrees Decimal Minutes": + converted = convDDToDDM(converted.degrees, precision); + break; + default: + throw "Unknown output format selection."; + } + if (returnRaw) { + return converted; + } else { + return converted.string; + } +} + +/** + * Split up the input using a space, and sanitise the result + * @param {string} input - The input data to be split + * @returns {number[]} An array of the different items in the string, stored as floats + */ +function splitInput (input){ + const split = []; + + input.split(" ").forEach(item => { + // Remove any character that isn't a digit + item = item.replace(/[^0-9.-]/g, ""); + if (item.length > 0){ + split.push(parseFloat(item, 10)); + } + }); + return split; +} + +/** + * Convert Degrees Minutes Seconds to Decimal Degrees + * @param {number} degrees - The degrees of the input co-ordinates + * @param {number} minutes - The minutes of the input co-ordinates + * @param {number} seconds - The seconds of the input co-ordinates + * @param {number} precision - The precision the result should be rounded to + * @returns {{string: string, degrees: number}} An object containing the raw converted value (obj.degrees), and a formatted string version (obj.string) + */ +function convDMSToDD (degrees, minutes, seconds, precision){ + const converted = new Object(); + converted.degrees = degrees + (minutes / 60) + (seconds / 3600); + converted.string = (Math.round(converted.degrees * precision) / precision) + "°"; + return converted; +} + +/** + * Convert Decimal Degrees Minutes to Decimal Degrees + * @param {number} degrees - The input degrees to be converted + * @param {number} minutes - The input minutes to be converted + * @param {number} precision - The precision which the result should be rounded to + * @returns {{string: string, degrees: number}} An object containing the raw converted value (obj.degrees), and a formatted string version (obj.string) + */ +function convDDMToDD (degrees, minutes, precision) { + const converted = new Object(); + converted.degrees = degrees + minutes / 60; + converted.string = ((Math.round(converted.degrees * precision) / precision) + "°"); + return converted; +} + +/** + * Convert Decimal Degrees to Decimal Degrees + * Doesn't affect the input, just puts it into an object + * @param {number} degrees - The input degrees to be converted + * @param {number} precision - The precision which the result should be rounded to + * @returns {{string: string, degrees: number}} An object containing the raw converted value (obj.degrees), and a formatted string version (obj.string) + */ +function convDDToDD (degrees, precision) { + const converted = new Object(); + converted.degrees = degrees; + converted.string = Math.round(converted.degrees * precision) / precision + "°"; + return converted; +} + +/** + * Convert Decimal Degrees to Degrees Minutes Seconds + * @param {number} decDegrees - The input data to be converted + * @returns {{string: string, degrees: number, minutes: number, seconds: number}} An object containing the raw converted value as separate numbers (.degrees, .minutes, .seconds), and a formatted string version (obj.string) + */ +function convDDToDMS (decDegrees) { + const degrees = Math.floor(decDegrees); + const minutes = Math.floor(60 * (decDegrees - degrees)); + const seconds = Math.round(3600 * (decDegrees - degrees) - 60 * minutes); + + const converted = new Object(); + converted.degrees = degrees; + converted.minutes = minutes; + converted.seconds = seconds; + converted.string = degrees + "° " + minutes + "' " + seconds + "\""; + return converted; +} + +/** + * Convert Decimal Degrees to Degrees Decimal Minutes + * @param {number} decDegrees - The input degrees to be converted + * @param {number} precision - The precision the input data should be rounded to + * @returns {{string: string, degrees: number, minutes: number}} An object containing the raw converted value as separate numbers (.degrees, .minutes), and a formatted string version (obj.string) + */ +function convDDToDDM (decDegrees, precision) { + const degrees = Math.floor(decDegrees); + const minutes = decDegrees - degrees; + const decMinutes = Math.round((minutes * 60) * precision) / precision; + + const converted = new Object(); + converted.degrees = degrees; + converted.minutes = decMinutes; + converted.string = degrees + "° " + decMinutes + "'"; + return converted; +} From 8bba4b297391377eaa0dcd527353742a3f7a0e5e Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 12 Jan 2019 00:20:25 +0000 Subject: [PATCH 29/52] More speedrun stats (literally 10x faster) --- package-lock.json | 6 +++--- package.json | 2 +- src/core/operations/YARARules.mjs | 19 ++++++++++++------- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2b4d058b..6ba03db3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7756,9 +7756,9 @@ "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA=" }, "libyara-wasm": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.8.tgz", - "integrity": "sha512-ZB+Ya3bEBoanvde47X8RzqpMBHgrPxrTZIJ/UEoatVnOEy2he1IORuotdSkP2o73URRzHGN1jNWDIhTdfbZ3rQ==" + "version": "0.0.9", + "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.9.tgz", + "integrity": "sha512-GTR8GD7q2xCwRzQpoSVh31KPbH145tCKaU3DAnQMpkf4aKqjnVpQlq6gUjyDhvw8RPZ2tamyQad5hg42KsVvRw==" }, "livereload-js": { "version": "2.4.0", diff --git a/package.json b/package.json index 8e21a0a6..a18331d8 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,7 @@ "jsqr": "^1.1.1", "jsrsasign": "8.0.12", "kbpgp": "^2.0.82", - "libyara-wasm": "0.0.8", + "libyara-wasm": "0.0.9", "lodash": "^4.17.11", "loglevel": "^1.6.1", "loglevel-message-prefix": "^3.0.0", diff --git a/src/core/operations/YARARules.mjs b/src/core/operations/YARARules.mjs index 094d9f43..58b3576c 100644 --- a/src/core/operations/YARARules.mjs +++ b/src/core/operations/YARARules.mjs @@ -59,16 +59,22 @@ class YARARules extends Operation { * @returns {string} */ run(input, args) { + if (ENVIRONMENT_IS_WORKER()) + self.sendStatusMessage("Instantiating YARA."); const [rules, showStrings, showLengths, showMeta] = args; return new Promise((resolve, reject) => { Yara().then(yara => { + if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Converting data for YARA."); let matchString = ""; - const inpArr = new Uint8Array(input); - const inpVec = new yara.vectorChar(); - for (let i = 0; i < inpArr.length; i++) { - inpVec.push_back(inpArr[i]); - } - const resp = yara.run(inpVec, rules); + + const inpArr = new Uint8Array(input); // Turns out embind knows that JS uint8array <==> C++ std::string + + if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Running YARA matching."); + + const resp = yara.run(inpArr, rules); + + if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Processing data."); + if (resp.compileErrors.size() > 0) { for (let i = 0; i < resp.compileErrors.size(); i++) { const compileError = resp.compileErrors.get(i); @@ -102,7 +108,6 @@ class YARARules extends Operation { } } } - } resolve(matchString); }); From 68fbbb64dba87ef1d7edf2138870e8b98aa3e9d8 Mon Sep 17 00:00:00 2001 From: j433866 Date: Mon, 14 Jan 2019 11:49:57 +0000 Subject: [PATCH 30/52] Add new Convert co-ordinate format module. Also added autodetect of co-ordinate format / delimiter --- src/core/config/Categories.json | 1 + src/core/lib/ConvertCoordinates.mjs | 95 ++++++++ .../operations/ConvertCoordinateFormat.mjs | 227 ++++++++++++++++++ 3 files changed, 323 insertions(+) create mode 100644 src/core/operations/ConvertCoordinateFormat.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index e9fe3399..13ea76c9 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -210,6 +210,7 @@ "Convert mass", "Convert speed", "Convert data units", + "Convert co-ordinate format", "Parse UNIX file permissions", "Swap endianness", "Parse colour code", diff --git a/src/core/lib/ConvertCoordinates.mjs b/src/core/lib/ConvertCoordinates.mjs index 45ab0690..a9b3d551 100644 --- a/src/core/lib/ConvertCoordinates.mjs +++ b/src/core/lib/ConvertCoordinates.mjs @@ -208,3 +208,98 @@ function convDDToDDM (decDegrees, precision) { converted.string = degrees + "° " + decMinutes + "'"; return converted; } + +/** + * + * @param {string} input - The input data whose format we need to detect + * @param {string} delim - The delimiter separating the data in input + * @returns {string} The input format + */ +export function findFormat (input, delim) { + input = input.trim(); + let testData; + if (delim.includes("Direction")) { + const split = input.split(/[NnEeSsWw]/); + if (split.length > 0) { + if (split[0] === "") { + // Direction Preceding + testData = split[1]; + } else { + // Direction Following + testData = split[0]; + } + } + } else if (delim !== "") { + const split = input.split(delim); + if (!input.includes(delim)) { + testData = input; + } + if (split.length > 0) { + if (split[0] !== "") { + testData = split[0]; + } else if (split.length > 1) { + testData = split[1]; + } + } + } + + // Test MGRS and Geohash + if (input.split(" ").length === 1) { + const mgrsPattern = new RegExp(/^[0-9]{2}[C-HJ-NP-X]{2}[A-Z]+/); + const geohashPattern = new RegExp(/^[0123456789bcdefghjkmnpqrstuvwxyz]+$/); + if (mgrsPattern.test(input.toUpperCase())) { + return "Military Grid Reference System"; + } else if (geohashPattern.test(input.toLowerCase())) { + return "Geohash"; + } + } + + // Test DMS/DDM/DD formats + if (testData !== undefined) { + const split = splitInput(testData); + if (split.length === 3) { + // DMS + return "Degrees Minutes Seconds"; + } else if (split.length === 2) { + // DDM + return "Degrees Decimal Minutes"; + } else if (split.length === 1) { + return "Decimal Degrees"; + } + } + return null; +} + +/** + * Automatically find the delimeter type from the given input + * @param {string} input + * @returns {string} Delimiter type + */ +export function findDelim (input) { + input = input.trim(); + const delims = [",", ";", ":"]; + // Direction + const testDir = input.match(/[NnEeSsWw]/g); + if (testDir !== null && testDir.length > 0 && testDir.length < 3) { + // Possible direction + const splitInput = input.split(/[NnEeSsWw]/); + if (splitInput.length <= 3 && splitInput.length > 0) { + // One of the splits should be an empty string + if (splitInput[0] === "") { + return "Direction Preceding"; + } else if (splitInput[splitInput.length - 1] === "") { + return "Direction Following"; + } + } + } + for (let i = 0; i < delims.length; i++) { + const delim = delims[i]; + if (input.includes(delim)) { + const splitInput = input.split(delim); + if (splitInput.length <= 3 && splitInput.length > 0) { + return delim; + } + } + } + return null; +} diff --git a/src/core/operations/ConvertCoordinateFormat.mjs b/src/core/operations/ConvertCoordinateFormat.mjs new file mode 100644 index 00000000..5f336630 --- /dev/null +++ b/src/core/operations/ConvertCoordinateFormat.mjs @@ -0,0 +1,227 @@ +/** + * @author j433866 [j433866@gmail.com] + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; +import {FORMATS, convertCoordinates, convertSingleCoordinate, findDelim, findFormat} from "../lib/ConvertCoordinates"; +import Utils from "../Utils"; + +/** + * Convert co-ordinate format operation + */ +class ConvertCoordinateFormat extends Operation { + + /** + * ConvertCoordinateFormat constructor + */ + constructor() { + super(); + + this.name = "Convert co-ordinate format"; + this.module = "Hashing"; + this.description = "Convert geographical coordinates between different formats.

Currently supported formats:
  • Degrees Minutes Seconds (DMS)
  • Degrees Decimal Minutes (DDM)
  • Decimal Degrees (DD)
  • Geohash
  • Military Grid Reference System (MGRS)
"; + this.infoURL = "https://wikipedia.org/wiki/Geographic_coordinate_conversion"; + this.inputType = "string"; + this.outputType = "string"; + this.args = [ + { + "name": "Input Format", + "type": "option", + "value": ["Auto"].concat(FORMATS) + }, + { + "name": "Input Delimiter", + "type": "option", + "value": [ + "Auto", + "Direction Preceding", // Need better names + "Direction Following", + "\\n", + "Comma", + "Semi-colon", + "Colon" + ] + }, + { + "name": "Output Format", + "type": "option", + "value": FORMATS + }, + { + "name": "Output Delimiter", + "type": "option", + "value": [ + "Space", + "Direction Preceding", // Need better names + "Direction Following", + "\\n", + "Comma", + "Semi-colon", + "Colon" + ] + }, + { + "name": "Precision", + "type": "number", + "value": 3 + } + ]; + } + + /** + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + run(input, args) { + const outFormat = args[2], + outDelim = args[3], + precision = args[4]; + let inFormat = args[0], + inDelim = args[1], + inLat, + inLong, + outLat, + outLong, + latDir = "", + longDir = "", + outSeparator = " "; + + // Autodetect input delimiter + if (inDelim === "Auto") { + inDelim = findDelim(input); + log.error("DATA: " + input + " DELIM: " + inDelim); + if (inDelim === null) { + inDelim = ""; + // throw new OperationError("Could not automatically detect the input delimiter."); + } + } else if (!inDelim.includes("Direction")) { + // Get the actual delimiter from the regex + inDelim = String(Utils.regexRep(inDelim)).slice(1, 2); + } + if (inFormat === "Auto") { + inFormat = findFormat(input, inDelim); + log.error("DATA: " + input + " FORMAT: " + inFormat); + if (inFormat === null) { + throw new OperationError("Could not automatically detect the input"); + } + } + + if (inDelim === "" && (inFormat !== "Geohash" && inFormat !== "Military Grid Reference System")) { + throw new OperationError("Could not automatically detect the input delimiter."); + } + + // Prepare input data + if (inFormat === "Geohash" || inFormat === "Military Grid Reference System") { + // Geohash only has one value, so just use the input + inLat = input; + } else if (inDelim === "Direction Preceding") { + // Split on the compass directions + const splitInput = input.split(/[NnEeSsWw]/); + const dir = input.match(/[NnEeSsWw]/g); + if (splitInput.length > 1) { + inLat = splitInput[1]; + if (dir !== null) { + latDir = dir[0]; + } + if (splitInput.length > 2) { + inLong = splitInput[2]; + if (dir !== null && dir.length > 1) { + longDir = dir[1]; + } + } + } + } else if (inDelim === "Direction Following") { + // Split on the compass directions + const splitInput = input.split(/[NnEeSsWw]/); + if (splitInput.length >= 1) { + inLat = splitInput[0]; + if (splitInput.length >= 2) { + inLong = splitInput[1]; + } + } + } else { + // Split on the delimiter + const splitInput = input.split(inDelim); + log.error(splitInput); + if (splitInput.length > 0) { + inLat = splitInput[0]; + if (splitInput.length >= 2) { + inLong = splitInput[1]; + } + } + } + + if (inFormat !== "Geohash" && inFormat !== "Military Grid Reference System" && outDelim.includes("Direction")) { + // Match on compass directions, and store the first 2 matches for the output + const dir = input.match(/[NnEeSsWw]/g); + if (dir !== null) { + latDir = dir[0]; + if (dir.length > 1) { + longDir = dir[1]; + } + } + } else if (outDelim === "\\n") { + outSeparator = "\n"; + } else if (outDelim === "Space") { + outSeparator = " "; + } else if (!outDelim.includes("Direction")) { + // Cut out the regex syntax (/) from the delimiter + outSeparator = String(Utils.regexRep(outDelim)).slice(1, 2); + } + + // Convert the co-ordinates + if (inLat !== undefined) { + if (inLong === undefined) { + if (inFormat !== "Geohash" && inFormat !== "Military Grid Reference System") { + if (outFormat === "Geohash" || outFormat === "Military Grid Reference System"){ + throw new OperationError(`${outFormat} needs both a latitude and a longitude to be calculated`); + } + } + if (inFormat === "Geohash" || inFormat === "Military Grid Reference System") { + // Geohash conversion is in convertCoordinates despite needing + // only one input as it needs to output two values + [outLat, outLong] = convertCoordinates(inLat, inLat, inFormat, outFormat, precision); + } else { + outLat = convertSingleCoordinate(inLat, inFormat, outFormat, precision); + } + } else { + [outLat, outLong] = convertCoordinates(inLat, inLong, inFormat, outFormat, precision); + } + } else { + throw new OperationError("No co-ordinates were detected in the input."); + } + + // Output conversion results if successful + if (outLat !== undefined) { + let output = ""; + if (outDelim === "Direction Preceding" && outFormat !== "Geohash" && outFormat !== "Military Grid Reference System") { + output += latDir += " "; + } + output += outLat; + if (outDelim === "Direction Following" && outFormat !== "Geohash" && outFormat !== "Military Grid Reference System") { + output += " " + latDir; + } + output += outSeparator; + + if (outLong !== undefined && outFormat !== "Geohash" && outFormat !== "Military Grid Reference System") { + if (outDelim === "Direction Preceding") { + output += longDir + " "; + } + output += outLong; + if (outDelim === "Direction Following") { + output += " " + longDir; + } + output += outSeparator; + } + return output; + } else { + throw new OperationError("Co-ordinate conversion failed."); + } + } +} + +export default ConvertCoordinateFormat; From 8d1f668fc550f5795a02f5f157411ffe45da89e0 Mon Sep 17 00:00:00 2001 From: j433866 Date: Mon, 14 Jan 2019 11:56:27 +0000 Subject: [PATCH 31/52] Remove old Geohash modules --- src/core/config/Categories.json | 4 +-- src/core/operations/FromGeohash.mjs | 44 ------------------------ src/core/operations/ToGeohash.mjs | 53 ----------------------------- 3 files changed, 1 insertion(+), 100 deletions(-) delete mode 100644 src/core/operations/FromGeohash.mjs delete mode 100644 src/core/operations/ToGeohash.mjs diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index 13ea76c9..2224050c 100755 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -302,9 +302,7 @@ "Adler-32 Checksum", "CRC-16 Checksum", "CRC-32 Checksum", - "TCP/IP Checksum", - "To Geohash", - "From Geohash" + "TCP/IP Checksum" ] }, { diff --git a/src/core/operations/FromGeohash.mjs b/src/core/operations/FromGeohash.mjs deleted file mode 100644 index b70273da..00000000 --- a/src/core/operations/FromGeohash.mjs +++ /dev/null @@ -1,44 +0,0 @@ -/** - * @author gchq77703 [] - * @copyright Crown Copyright 2018 - * @license Apache-2.0 - */ - -import Operation from "../Operation"; -import geohash from "ngeohash"; - -/** - * From Geohash operation - */ -class FromGeohash extends Operation { - - /** - * FromGeohash constructor - */ - constructor() { - super(); - - this.name = "From Geohash"; - this.module = "Hashing"; - this.description = "Converts Geohash strings into Lat/Long coordinates. For example, ww8p1r4t8 becomes 37.8324,112.5584."; - this.infoURL = "https://wikipedia.org/wiki/Geohash"; - this.inputType = "string"; - this.outputType = "string"; - this.args = []; - } - - /** - * @param {string} input - * @param {Object[]} args - * @returns {string} - */ - run(input, args) { - return input.split("\n").map(line => { - const coords = geohash.decode(line); - return [coords.latitude, coords.longitude].join(","); - }).join("\n"); - } - -} - -export default FromGeohash; diff --git a/src/core/operations/ToGeohash.mjs b/src/core/operations/ToGeohash.mjs deleted file mode 100644 index 0e7f53ac..00000000 --- a/src/core/operations/ToGeohash.mjs +++ /dev/null @@ -1,53 +0,0 @@ -/** - * @author gchq77703 [] - * @copyright Crown Copyright 2018 - * @license Apache-2.0 - */ - -import Operation from "../Operation"; -import geohash from "ngeohash"; - -/** - * To Geohash operation - */ -class ToGeohash extends Operation { - - /** - * ToGeohash constructor - */ - constructor() { - super(); - - this.name = "To Geohash"; - this.module = "Hashing"; - this.description = "Converts Lat/Long coordinates into a Geohash string. For example, 37.8324,112.5584 becomes ww8p1r4t8."; - this.infoURL = "https://wikipedia.org/wiki/Geohash"; - this.inputType = "string"; - this.outputType = "string"; - this.args = [ - { - name: "Precision", - type: "number", - value: 9 - } - ]; - } - - /** - * @param {string} input - * @param {Object[]} args - * @returns {string} - */ - run(input, args) { - const [precision] = args; - - return input.split("\n").map(line => { - line = line.replace(/ /g, ""); - if (line === "") return ""; - return geohash.encode(...line.split(",").map(num => parseFloat(num)), precision); - }).join("\n"); - } - -} - -export default ToGeohash; From 8b77ad77480b7cf1450e32d2c63dc17f0a6f46ee Mon Sep 17 00:00:00 2001 From: j433866 Date: Mon, 14 Jan 2019 12:49:28 +0000 Subject: [PATCH 32/52] Stop delimiters breaking MGRS conversion --- src/core/operations/ConvertCoordinateFormat.mjs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/operations/ConvertCoordinateFormat.mjs b/src/core/operations/ConvertCoordinateFormat.mjs index 5f336630..d24b2135 100644 --- a/src/core/operations/ConvertCoordinateFormat.mjs +++ b/src/core/operations/ConvertCoordinateFormat.mjs @@ -117,7 +117,8 @@ class ConvertCoordinateFormat extends Operation { // Prepare input data if (inFormat === "Geohash" || inFormat === "Military Grid Reference System") { // Geohash only has one value, so just use the input - inLat = input; + // Replace anything that isn't a valid character in Geohash / MGRS + inLat = input.replace(/[^A-Za-z0-9]/, ""); } else if (inDelim === "Direction Preceding") { // Split on the compass directions const splitInput = input.split(/[NnEeSsWw]/); From 1a88a0164cf8d27d260afc9e8b8b72d5db39fe3a Mon Sep 17 00:00:00 2001 From: j433866 Date: Mon, 14 Jan 2019 13:00:14 +0000 Subject: [PATCH 33/52] Fix delimiter breaking Geohash detection --- src/core/lib/ConvertCoordinates.mjs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/core/lib/ConvertCoordinates.mjs b/src/core/lib/ConvertCoordinates.mjs index a9b3d551..9b9d8881 100644 --- a/src/core/lib/ConvertCoordinates.mjs +++ b/src/core/lib/ConvertCoordinates.mjs @@ -45,6 +45,7 @@ export function convertCoordinates (inLat, inLong, inFormat, outFormat, precisio convLong = convertSingleCoordinate(inLong, inFormat, "Decimal Degrees", 15).split("°"); } + // Convert Geohash and MGRS here, as they need both the lat and long values if (outFormat === "Geohash") { convLat = geohash.encode(parseFloat(convLat), parseFloat(convLong), precision); } else if (outFormat === "Military Grid Reference System") { @@ -244,12 +245,14 @@ export function findFormat (input, delim) { } // Test MGRS and Geohash - if (input.split(" ").length === 1) { + if (input.split(" ").length <= 1) { + const filteredInput = input.replace(/[^A-Za-z0-9]/, "").toUpperCase(); const mgrsPattern = new RegExp(/^[0-9]{2}[C-HJ-NP-X]{2}[A-Z]+/); - const geohashPattern = new RegExp(/^[0123456789bcdefghjkmnpqrstuvwxyz]+$/); - if (mgrsPattern.test(input.toUpperCase())) { + const geohashPattern = new RegExp(/^[0123456789BCDEFGHJKMNPQRSTUVWXYZ]+$/); + log.error(filteredInput); + if (mgrsPattern.test(filteredInput)) { return "Military Grid Reference System"; - } else if (geohashPattern.test(input.toLowerCase())) { + } else if (geohashPattern.test(filteredInput)) { return "Geohash"; } } @@ -292,6 +295,8 @@ export function findDelim (input) { } } } + + // Loop through the standard delimiters, and try to find them in the input for (let i = 0; i < delims.length; i++) { const delim = delims[i]; if (input.includes(delim)) { From b3ac8d0835e90c9301cfd86f58d2f619a1d246c6 Mon Sep 17 00:00:00 2001 From: j433866 Date: Mon, 14 Jan 2019 13:49:49 +0000 Subject: [PATCH 34/52] Removed some debug logging --- src/core/lib/ConvertCoordinates.mjs | 1 - src/core/operations/ConvertCoordinateFormat.mjs | 8 ++------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/core/lib/ConvertCoordinates.mjs b/src/core/lib/ConvertCoordinates.mjs index 9b9d8881..0078cfb7 100644 --- a/src/core/lib/ConvertCoordinates.mjs +++ b/src/core/lib/ConvertCoordinates.mjs @@ -249,7 +249,6 @@ export function findFormat (input, delim) { const filteredInput = input.replace(/[^A-Za-z0-9]/, "").toUpperCase(); const mgrsPattern = new RegExp(/^[0-9]{2}[C-HJ-NP-X]{2}[A-Z]+/); const geohashPattern = new RegExp(/^[0123456789BCDEFGHJKMNPQRSTUVWXYZ]+$/); - log.error(filteredInput); if (mgrsPattern.test(filteredInput)) { return "Military Grid Reference System"; } else if (geohashPattern.test(filteredInput)) { diff --git a/src/core/operations/ConvertCoordinateFormat.mjs b/src/core/operations/ConvertCoordinateFormat.mjs index d24b2135..7528fccc 100644 --- a/src/core/operations/ConvertCoordinateFormat.mjs +++ b/src/core/operations/ConvertCoordinateFormat.mjs @@ -22,7 +22,7 @@ class ConvertCoordinateFormat extends Operation { this.name = "Convert co-ordinate format"; this.module = "Hashing"; - this.description = "Convert geographical coordinates between different formats.

Currently supported formats:
  • Degrees Minutes Seconds (DMS)
  • Degrees Decimal Minutes (DDM)
  • Decimal Degrees (DD)
  • Geohash
  • Military Grid Reference System (MGRS)
"; + this.description = "Convert geographical coordinates between different formats.

Supported formats:
  • Degrees Minutes Seconds (DMS)
  • Degrees Decimal Minutes (DDM)
  • Decimal Degrees (DD)
  • Geohash
  • Military Grid Reference System (MGRS)
"; this.infoURL = "https://wikipedia.org/wiki/Geographic_coordinate_conversion"; this.inputType = "string"; this.outputType = "string"; @@ -93,10 +93,8 @@ class ConvertCoordinateFormat extends Operation { // Autodetect input delimiter if (inDelim === "Auto") { inDelim = findDelim(input); - log.error("DATA: " + input + " DELIM: " + inDelim); if (inDelim === null) { inDelim = ""; - // throw new OperationError("Could not automatically detect the input delimiter."); } } else if (!inDelim.includes("Direction")) { // Get the actual delimiter from the regex @@ -104,9 +102,8 @@ class ConvertCoordinateFormat extends Operation { } if (inFormat === "Auto") { inFormat = findFormat(input, inDelim); - log.error("DATA: " + input + " FORMAT: " + inFormat); if (inFormat === null) { - throw new OperationError("Could not automatically detect the input"); + throw new OperationError("Could not automatically detect the input format."); } } @@ -147,7 +144,6 @@ class ConvertCoordinateFormat extends Operation { } else { // Split on the delimiter const splitInput = input.split(inDelim); - log.error(splitInput); if (splitInput.length > 0) { inLat = splitInput[0]; if (splitInput.length >= 2) { From 04b0b8c72344aa136dd2cd879fc99f2ffb678ec5 Mon Sep 17 00:00:00 2001 From: j433866 Date: Mon, 14 Jan 2019 14:58:41 +0000 Subject: [PATCH 35/52] Tidy up code --- src/core/lib/ConvertCoordinates.mjs | 21 +++++++++---------- .../operations/ConvertCoordinateFormat.mjs | 4 ++-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/core/lib/ConvertCoordinates.mjs b/src/core/lib/ConvertCoordinates.mjs index 0078cfb7..1c6ae7e2 100644 --- a/src/core/lib/ConvertCoordinates.mjs +++ b/src/core/lib/ConvertCoordinates.mjs @@ -259,14 +259,13 @@ export function findFormat (input, delim) { // Test DMS/DDM/DD formats if (testData !== undefined) { const split = splitInput(testData); - if (split.length === 3) { - // DMS - return "Degrees Minutes Seconds"; - } else if (split.length === 2) { - // DDM - return "Degrees Decimal Minutes"; - } else if (split.length === 1) { - return "Decimal Degrees"; + switch (split.length){ + case 3: + return "Degrees Minutes Seconds"; + case 2: + return "Degrees Decimal Minutes"; + case 1: + return "Decimal Degrees"; } } return null; @@ -280,13 +279,12 @@ export function findFormat (input, delim) { export function findDelim (input) { input = input.trim(); const delims = [",", ";", ":"]; - // Direction const testDir = input.match(/[NnEeSsWw]/g); if (testDir !== null && testDir.length > 0 && testDir.length < 3) { - // Possible direction + // Possibly contains a direction const splitInput = input.split(/[NnEeSsWw]/); if (splitInput.length <= 3 && splitInput.length > 0) { - // One of the splits should be an empty string + // If there's 3 splits (one should be empty), then assume we have directions if (splitInput[0] === "") { return "Direction Preceding"; } else if (splitInput[splitInput.length - 1] === "") { @@ -301,6 +299,7 @@ export function findDelim (input) { if (input.includes(delim)) { const splitInput = input.split(delim); if (splitInput.length <= 3 && splitInput.length > 0) { + // Don't want to try and convert more than 2 co-ordinates return delim; } } diff --git a/src/core/operations/ConvertCoordinateFormat.mjs b/src/core/operations/ConvertCoordinateFormat.mjs index 7528fccc..b9a9766c 100644 --- a/src/core/operations/ConvertCoordinateFormat.mjs +++ b/src/core/operations/ConvertCoordinateFormat.mjs @@ -37,7 +37,7 @@ class ConvertCoordinateFormat extends Operation { "type": "option", "value": [ "Auto", - "Direction Preceding", // Need better names + "Direction Preceding", "Direction Following", "\\n", "Comma", @@ -55,7 +55,7 @@ class ConvertCoordinateFormat extends Operation { "type": "option", "value": [ "Space", - "Direction Preceding", // Need better names + "Direction Preceding", "Direction Following", "\\n", "Comma", From ee360521bb757c9cae9010b13db0932a46aa1e02 Mon Sep 17 00:00:00 2001 From: j433866 Date: Mon, 14 Jan 2019 16:41:06 +0000 Subject: [PATCH 36/52] Remove MGRS npm module --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index 92fda38b..414e4d50 100644 --- a/package.json +++ b/package.json @@ -104,7 +104,6 @@ "lodash": "^4.17.11", "loglevel": "^1.6.1", "loglevel-message-prefix": "^3.0.0", - "mgrs": "^1.0.0", "moment": "^2.22.2", "moment-timezone": "^0.5.23", "ngeohash": "^0.6.0", From 4d8127a7d98b18309d5d4bc44892627f647adb23 Mon Sep 17 00:00:00 2001 From: Callum Fraser Date: Mon, 14 Jan 2019 22:24:32 +0000 Subject: [PATCH 37/52] Modified description of ToBase64 operation Addresses #472 --- src/core/operations/ToBase64.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/ToBase64.mjs b/src/core/operations/ToBase64.mjs index f9aa569b..cffc3140 100644 --- a/src/core/operations/ToBase64.mjs +++ b/src/core/operations/ToBase64.mjs @@ -20,7 +20,7 @@ class ToBase64 extends Operation { this.name = "To Base64"; this.module = "Default"; - this.description = "Base64 is a notation for encoding arbitrary byte data using a restricted set of symbols that can be conveniently used by humans and processed by computers.

This operation decodes data from an ASCII Base64 string back into its raw format.

e.g. aGVsbG8= becomes hello"; + this.description = "Base64 is a notation for encoding arbitrary byte data using a restricted set of symbols that can be conveniently used by humans and processed by computers.

This operation encodes raw data into an ASCII Base64 string.

e.g. hello becomes aGVsbG8="; this.infoURL = "https://wikipedia.org/wiki/Base64"; this.inputType = "ArrayBuffer"; this.outputType = "string"; From ad4451a757f0650ee492c41a89014b36ead8993b Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 15 Jan 2019 10:13:11 +0000 Subject: [PATCH 38/52] Rewrite MGRS to use new Geodesy module. Added Ordnance Survey grid reference support --- package.json | 1 + src/core/lib/ConvertCoordinates.mjs | 43 ++++++++++++++++--- .../operations/ConvertCoordinateFormat.mjs | 25 +++++------ 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/package.json b/package.json index 414e4d50..4e38e4fa 100644 --- a/package.json +++ b/package.json @@ -91,6 +91,7 @@ "esprima": "^4.0.1", "exif-parser": "^0.1.12", "file-saver": "^2.0.0-rc.4", + "geodesy": "^1.1.3", "highlight.js": "^9.13.1", "jquery": "^3.3.1", "js-crc": "^0.2.0", diff --git a/src/core/lib/ConvertCoordinates.mjs b/src/core/lib/ConvertCoordinates.mjs index 1c6ae7e2..06e40f02 100644 --- a/src/core/lib/ConvertCoordinates.mjs +++ b/src/core/lib/ConvertCoordinates.mjs @@ -7,7 +7,7 @@ */ import geohash from "ngeohash"; -import mgrs from "mgrs"; +import geodesy from "geodesy"; /** * Co-ordinate formats @@ -17,7 +17,19 @@ export const FORMATS = [ "Degrees Decimal Minutes", "Decimal Degrees", "Geohash", - "Military Grid Reference System" + "Military Grid Reference System", + "Ordnance Survey National Grid" +]; + +/** + * Formats that are made up of one string + * These formats skip bits like filtering delimiters and + * are outputted differently (only one output) + */ +export const STRING_FORMATS = [ + "Geohash", + "Military Grid Reference System", + "Ordnance Survey National Grid" ]; /** @@ -37,9 +49,22 @@ export function convertCoordinates (inLat, inLong, inFormat, outFormat, precisio convLat = hash.latitude.toString(); convLong = hash.longitude.toString(); } else if (inFormat === "Military Grid Reference System") { - const result = mgrs.toPoint(inLat.replace(" ", "")); - convLat = result[1]; - convLong = result[0]; + const utm = geodesy.Mgrs.parse(inLat).toUtm(); + const result = utm.toLatLonE().toString("d", 4).replace(/[^0-9.,]/g, ""); + const splitResult = result.split(","); + if (splitResult.length === 2) { + convLat = splitResult[0]; + convLong = splitResult[1]; + } + } else if (inFormat === "Ordnance Survey National Grid") { + const osng = geodesy.OsGridRef.parse(inLat); + const latlon = geodesy.OsGridRef.osGridToLatLon(osng, geodesy.LatLonEllipsoidal.datum.WGS84); + const result = latlon.toString("d", 4).replace(/[^0-9.,]/g, ""); + const splitResult = result.split(","); + if (splitResult.length === 2) { + convLat = splitResult[0]; + convLong = splitResult[1]; + } } else { convLat = convertSingleCoordinate(inLat, inFormat, "Decimal Degrees", 15).split("°"); convLong = convertSingleCoordinate(inLong, inFormat, "Decimal Degrees", 15).split("°"); @@ -49,7 +74,13 @@ export function convertCoordinates (inLat, inLong, inFormat, outFormat, precisio if (outFormat === "Geohash") { convLat = geohash.encode(parseFloat(convLat), parseFloat(convLong), precision); } else if (outFormat === "Military Grid Reference System") { - convLat = mgrs.forward([parseFloat(convLong), parseFloat(convLat)], precision); + const utm = new geodesy.LatLonEllipsoidal(parseFloat(convLat), parseFloat(convLong)).toUtm(); + const mgrs = utm.toMgrs(); + convLat = mgrs.toString(); + } else if (outFormat === "Ordnance Survey National Grid") { + const latlon = new geodesy.LatLonEllipsoidal(parseFloat(convLat), parseFloat(convLong)); + const osng = geodesy.OsGridRef.latLonToOsGrid(latlon); + convLat = osng.toString(); } else { convLat = convertSingleCoordinate(convLat.toString(), "Decimal Degrees", outFormat, precision); convLong = convertSingleCoordinate(convLong.toString(), "Decimal Degrees", outFormat, precision); diff --git a/src/core/operations/ConvertCoordinateFormat.mjs b/src/core/operations/ConvertCoordinateFormat.mjs index b9a9766c..afc95982 100644 --- a/src/core/operations/ConvertCoordinateFormat.mjs +++ b/src/core/operations/ConvertCoordinateFormat.mjs @@ -6,7 +6,7 @@ import Operation from "../Operation"; import OperationError from "../errors/OperationError"; -import {FORMATS, convertCoordinates, convertSingleCoordinate, findDelim, findFormat} from "../lib/ConvertCoordinates"; +import {FORMATS, STRING_FORMATS, convertCoordinates, convertSingleCoordinate, findDelim, findFormat} from "../lib/ConvertCoordinates"; import Utils from "../Utils"; /** @@ -22,7 +22,7 @@ class ConvertCoordinateFormat extends Operation { this.name = "Convert co-ordinate format"; this.module = "Hashing"; - this.description = "Convert geographical coordinates between different formats.

Supported formats:
  • Degrees Minutes Seconds (DMS)
  • Degrees Decimal Minutes (DDM)
  • Decimal Degrees (DD)
  • Geohash
  • Military Grid Reference System (MGRS)
"; + this.description = "Convert geographical coordinates between different formats.

Supported formats:
  • Degrees Minutes Seconds (DMS)
  • Degrees Decimal Minutes (DDM)
  • Decimal Degrees (DD)
  • Geohash
  • Military Grid Reference System (MGRS)
  • Ordnance Survey National Grid (OSNG)
"; this.infoURL = "https://wikipedia.org/wiki/Geographic_coordinate_conversion"; this.inputType = "string"; this.outputType = "string"; @@ -107,14 +107,14 @@ class ConvertCoordinateFormat extends Operation { } } - if (inDelim === "" && (inFormat !== "Geohash" && inFormat !== "Military Grid Reference System")) { + if (inDelim === "" && (!STRING_FORMATS.includes(inFormat))) { throw new OperationError("Could not automatically detect the input delimiter."); } // Prepare input data - if (inFormat === "Geohash" || inFormat === "Military Grid Reference System") { + if (STRING_FORMATS.includes(inFormat)) { // Geohash only has one value, so just use the input - // Replace anything that isn't a valid character in Geohash / MGRS + // Replace anything that isn't a valid character in Geohash / MGRS / OSNG inLat = input.replace(/[^A-Za-z0-9]/, ""); } else if (inDelim === "Direction Preceding") { // Split on the compass directions @@ -152,7 +152,7 @@ class ConvertCoordinateFormat extends Operation { } } - if (inFormat !== "Geohash" && inFormat !== "Military Grid Reference System" && outDelim.includes("Direction")) { + if (!STRING_FORMATS.includes(inFormat) && outDelim.includes("Direction")) { // Match on compass directions, and store the first 2 matches for the output const dir = input.match(/[NnEeSsWw]/g); if (dir !== null) { @@ -173,14 +173,15 @@ class ConvertCoordinateFormat extends Operation { // Convert the co-ordinates if (inLat !== undefined) { if (inLong === undefined) { - if (inFormat !== "Geohash" && inFormat !== "Military Grid Reference System") { - if (outFormat === "Geohash" || outFormat === "Military Grid Reference System"){ + if (!STRING_FORMATS.includes(inFormat)) { + if (STRING_FORMATS.includes(outFormat)){ throw new OperationError(`${outFormat} needs both a latitude and a longitude to be calculated`); } } - if (inFormat === "Geohash" || inFormat === "Military Grid Reference System") { + if (STRING_FORMATS.includes(inFormat)) { // Geohash conversion is in convertCoordinates despite needing // only one input as it needs to output two values + inLat = inLat.replace(/[^A-Za-z0-9]/g, ""); [outLat, outLong] = convertCoordinates(inLat, inLat, inFormat, outFormat, precision); } else { outLat = convertSingleCoordinate(inLat, inFormat, outFormat, precision); @@ -195,16 +196,16 @@ class ConvertCoordinateFormat extends Operation { // Output conversion results if successful if (outLat !== undefined) { let output = ""; - if (outDelim === "Direction Preceding" && outFormat !== "Geohash" && outFormat !== "Military Grid Reference System") { + if (outDelim === "Direction Preceding" && !STRING_FORMATS.includes(outFormat)) { output += latDir += " "; } output += outLat; - if (outDelim === "Direction Following" && outFormat !== "Geohash" && outFormat !== "Military Grid Reference System") { + if (outDelim === "Direction Following" && !STRING_FORMATS.includes(outFormat)) { output += " " + latDir; } output += outSeparator; - if (outLong !== undefined && outFormat !== "Geohash" && outFormat !== "Military Grid Reference System") { + if (outLong !== undefined && !STRING_FORMATS.includes(outFormat)) { if (outDelim === "Direction Preceding") { output += longDir + " "; } From 5e68959c03cc460edd93d7dbe9444d8250958711 Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 15 Jan 2019 10:25:49 +0000 Subject: [PATCH 39/52] Catch when OS grid references aren't calculated --- src/core/lib/ConvertCoordinates.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/core/lib/ConvertCoordinates.mjs b/src/core/lib/ConvertCoordinates.mjs index 06e40f02..b6f9e9bf 100644 --- a/src/core/lib/ConvertCoordinates.mjs +++ b/src/core/lib/ConvertCoordinates.mjs @@ -81,6 +81,9 @@ export function convertCoordinates (inLat, inLong, inFormat, outFormat, precisio const latlon = new geodesy.LatLonEllipsoidal(parseFloat(convLat), parseFloat(convLong)); const osng = geodesy.OsGridRef.latLonToOsGrid(latlon); convLat = osng.toString(); + if (convLat === "") { + throw "Couldn't convert co-ordinates to Ordnance Survey National Grid. Are they out of range?"; + } } else { convLat = convertSingleCoordinate(convLat.toString(), "Decimal Degrees", outFormat, precision); convLong = convertSingleCoordinate(convLong.toString(), "Decimal Degrees", outFormat, precision); From d00b0f4c0e9e06af134de7cf680425b2be3668d9 Mon Sep 17 00:00:00 2001 From: j433866 Date: Tue, 15 Jan 2019 15:55:49 +0000 Subject: [PATCH 40/52] Basically rewrote the whole thing using the new geodesy module --- src/core/lib/ConvertCoordinates.mjs | 464 +++++++++++++----- .../operations/ConvertCoordinateFormat.mjs | 162 +----- 2 files changed, 353 insertions(+), 273 deletions(-) diff --git a/src/core/lib/ConvertCoordinates.mjs b/src/core/lib/ConvertCoordinates.mjs index b6f9e9bf..cec2439c 100644 --- a/src/core/lib/ConvertCoordinates.mjs +++ b/src/core/lib/ConvertCoordinates.mjs @@ -18,148 +18,240 @@ export const FORMATS = [ "Decimal Degrees", "Geohash", "Military Grid Reference System", - "Ordnance Survey National Grid" + "Ordnance Survey National Grid", + "Universal Transverse Mercator" ]; /** - * Formats that are made up of one string - * These formats skip bits like filtering delimiters and - * are outputted differently (only one output) + * Formats that should be passed to Geodesy module as-is + * Spaces are still removed */ -export const STRING_FORMATS = [ +const NO_CHANGE = [ "Geohash", "Military Grid Reference System", - "Ordnance Survey National Grid" + "Ordnance Survey National Grid", + "Universal Transverse Mercator", ]; /** * Convert a given latitude and longitude into a different format. - * @param {string} inLat - Input latitude to be converted. Use this for supplying single values for conversion (e.g. geohash) - * @param {string} inLong - Input longitude to be converted + * @param {string} input - Input string to be converted * @param {string} inFormat - Format of the input coordinates + * @param {string} inDelim - The delimiter splitting the lat/long of the input * @param {string} outFormat - Format to convert to + * @param {string} outDelim - The delimiter to separate the output with + * @param {string} includeDir - Whether or not to include the compass direction in the output * @param {number} precision - Precision of the result - * @returns {string[]} Array containing the converted latitude and longitude + * @returns {string} A formatted string of the converted co-ordinates */ -export function convertCoordinates (inLat, inLong, inFormat, outFormat, precision) { - let convLat = inLat; - let convLong = inLong; +export function convertCoordinates (input, inFormat, inDelim, outFormat, outDelim, includeDir, precision) { + let isPair = false, + split, + latlon, + conv, + inLatDir, + inLongDir; + + if (inDelim === "Auto") { + inDelim = findDelim(input); + } else { + inDelim = realDelim(inDelim); + } + if (inFormat === "Auto") { + inFormat = findFormat(input, inDelim); + if (inFormat === null) { + throw "Unable to detect the input format automatically."; + } + } + if (inDelim === null && !inFormat.includes("Direction")) { + throw "Unable to detect the input delimiter automatically."; + } + outDelim = realDelim(outDelim); + + if (!NO_CHANGE.includes(inFormat)) { + split = input.split(inDelim); + if (split.length > 1) { + isPair = true; + } + } else { + input = input.replace(inDelim, ""); + isPair = true; + } + + if (inFormat.includes("Degrees")) { + [inLatDir, inLongDir] = findDirs(input, inDelim); + } + if (inFormat === "Geohash") { - const hash = geohash.decode(inLat); - convLat = hash.latitude.toString(); - convLong = hash.longitude.toString(); + const hash = geohash.decode(input.replace(/[^A-Za-z0-9]/g, "")); + latlon = new geodesy.LatLonEllipsoidal(hash.latitude, hash.longitude); } else if (inFormat === "Military Grid Reference System") { - const utm = geodesy.Mgrs.parse(inLat).toUtm(); - const result = utm.toLatLonE().toString("d", 4).replace(/[^0-9.,]/g, ""); - const splitResult = result.split(","); - if (splitResult.length === 2) { - convLat = splitResult[0]; - convLong = splitResult[1]; - } + const utm = geodesy.Mgrs.parse(input.replace(/[^A-Za-z0-9]/g, "")).toUtm(); + latlon = utm.toLatLonE(); } else if (inFormat === "Ordnance Survey National Grid") { - const osng = geodesy.OsGridRef.parse(inLat); - const latlon = geodesy.OsGridRef.osGridToLatLon(osng, geodesy.LatLonEllipsoidal.datum.WGS84); - const result = latlon.toString("d", 4).replace(/[^0-9.,]/g, ""); - const splitResult = result.split(","); - if (splitResult.length === 2) { - convLat = splitResult[0]; - convLong = splitResult[1]; + const osng = geodesy.OsGridRef.parse(input.replace(/[^A-Za-z0-9]/g, "")); + latlon = geodesy.OsGridRef.osGridToLatLon(osng); + } else if (inFormat === "Universal Transverse Mercator") { + if (/^[\d]{2}[A-Za-z]/.test(input)) { + input = input.slice(0, 2) + " " + input.slice(2); + } + const utm = geodesy.Utm.parse(input); + latlon = utm.toLatLonE(); + } else if (inFormat === "Degrees Minutes Seconds") { + if (isPair) { + split[0] = split[0].replace(/[NnEeSsWw]/g, "").trim(); + split[1] = split[1].replace(/[NnEeSsWw]/g, "").trim(); + const splitLat = split[0].split(/[°′″'"\s]/g), + splitLong = split[1].split(/[°′″'"\s]/g); + + if (splitLat.length >= 3 && splitLong.length >= 3) { + const lat = convDMSToDD(parseFloat(splitLat[0]), parseFloat(splitLat[1]), parseFloat(splitLat[2]), 10); + const long = convDMSToDD(parseFloat(splitLong[0]), parseFloat(splitLong[1]), parseFloat(splitLong[2]), 10); + latlon = new geodesy.LatLonEllipsoidal(lat.degrees, long.degrees); + } + } else { + // Create a new latlon object anyway, but we can ignore the lon value + split[0] = split[0].replace(/[NnEeSsWw]/g, "").trim(); + const splitLat = split[0].split(/[°′″'"\s]/g); + if (splitLat.length >= 3) { + const lat = convDMSToDD(parseFloat(splitLat[0]), parseFloat(splitLat[1]), parseFloat(splitLat[2])); + latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lat.degrees); + } + } + } else if (inFormat === "Degrees Decimal Minutes") { + if (isPair) { + const splitLat = splitInput(split[0]); + const splitLong = splitInput(split[1]); + if (splitLat.length !== 2 || splitLong.length !== 2) { + throw "Invalid co-ordinate format for Degrees Decimal Minutes."; + } + const lat = convDDMToDD(splitLat[0], splitLat[1], 10); + const long = convDDMToDD(splitLong[0], splitLong[1], 10); + latlon = new geodesy.LatLonEllipsoidal(lat.degrees, long.degrees); + } else { + const splitLat = splitInput(input); + if (splitLat.length !== 2) { + throw "Invalid co-ordinate format for Degrees Decimal Minutes."; + } + const lat = convDDMToDD(splitLat[0], splitLat[1], 10); + latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lat.degrees); + } + } else if (inFormat === "Decimal Degrees") { + if (isPair) { + const splitLat = splitInput(split[0]); + const splitLong = splitInput(split[1]); + if (splitLat.length !== 1 || splitLong.length !== 1) { + throw "Invalid co-ordinate format for Decimal Degrees."; + } + latlon = new geodesy.LatLonEllipsoidal(splitLat[0], splitLong[0]); + } else { + const splitLat = splitInput(split[0]); + if (splitLat.length !== 1) { + throw "Invalid co-ordinate format for Decimal Degrees."; + } + latlon = new geodesy.LatLonEllipsoidal(splitLat[0], splitLat[0]); } } else { - convLat = convertSingleCoordinate(inLat, inFormat, "Decimal Degrees", 15).split("°"); - convLong = convertSingleCoordinate(inLong, inFormat, "Decimal Degrees", 15).split("°"); + throw "Invalid input co-ordinate format selected."; } - // Convert Geohash and MGRS here, as they need both the lat and long values - if (outFormat === "Geohash") { - convLat = geohash.encode(parseFloat(convLat), parseFloat(convLong), precision); + // Everything is now a geodesy latlon object + if (outFormat === "Decimal Degrees") { + conv = latlon.toString("d", precision); + if (!isPair) { + conv = conv.split(",")[0]; + } + } else if (outFormat === "Degrees Decimal Minutes") { + conv = latlon.toString("dm", precision); + if (!isPair) { + conv = conv.split(",")[0]; + } + } else if (outFormat === "Degrees Minutes Seconds") { + conv = latlon.toString("dms", precision); + if (!isPair) { + conv = conv.split(",")[0]; + } + } else if (outFormat === "Geohash") { + conv = geohash.encode(latlon.lat.toString(), latlon.lon.toString(), precision); } else if (outFormat === "Military Grid Reference System") { - const utm = new geodesy.LatLonEllipsoidal(parseFloat(convLat), parseFloat(convLong)).toUtm(); + const utm = latlon.toUtm(); const mgrs = utm.toMgrs(); - convLat = mgrs.toString(); + conv = mgrs.toString(precision); } else if (outFormat === "Ordnance Survey National Grid") { - const latlon = new geodesy.LatLonEllipsoidal(parseFloat(convLat), parseFloat(convLong)); const osng = geodesy.OsGridRef.latLonToOsGrid(latlon); - convLat = osng.toString(); - if (convLat === "") { - throw "Couldn't convert co-ordinates to Ordnance Survey National Grid. Are they out of range?"; + if (osng.toString() === "") { + throw "Could not convert co-ordinates to OS National Grid. Are the co-ordinates in range?"; } - } else { - convLat = convertSingleCoordinate(convLat.toString(), "Decimal Degrees", outFormat, precision); - convLong = convertSingleCoordinate(convLong.toString(), "Decimal Degrees", outFormat, precision); + conv = osng.toString(precision); + } else if (outFormat === "Universal Transverse Mercator") { + const utm = latlon.toUtm(); + conv = utm.toString(precision); } - return [convLat, convLong]; + if (conv === undefined) { + throw "Error converting co-ordinates."; + } + if (outFormat.includes("Degrees")) { + let [latDir, longDir] = findDirs(conv, outDelim); + if (inLatDir !== undefined) { + latDir = inLatDir; + } + if (inLongDir !== undefined) { + longDir = inLongDir; + } + // DMS/DDM/DD + conv = conv.replace(", ", outDelim); + // Remove any directions from the current string, + // so we can put them where we want them + conv = conv.replace(/[NnEeSsWw]/g, ""); + if (includeDir !== "None") { + let outConv = ""; + if (!isPair) { + if (includeDir === "Before") { + outConv += latDir + " " + conv; + } else { + outConv += conv + " " + latDir; + } + } else { + const splitConv = conv.split(outDelim); + if (splitConv.length === 2) { + if (includeDir === "Before") { + outConv += latDir + " "; + } + outConv += splitConv[0]; + if (includeDir === "After") { + outConv += " " + latDir; + } + outConv += outDelim; + if (includeDir === "Before") { + outConv += longDir + " "; + } + outConv += splitConv[1]; + if (includeDir === "After") { + outConv += " " + longDir; + } + } + } + conv = outConv; + } + } + + return conv; } /** - * @param {string} input - The input co-ordinate to be converted - * @param {string} inFormat - The format of the input co-ordinates - * @param {string} outFormat - The format which input should be converted to - * @param {boolean} returnRaw - When true, returns the raw float instead of a String - * @returns {string|{Object}} The converted co-ordinate result, as either the raw object or a formatted string - */ -export function convertSingleCoordinate (input, inFormat, outFormat, precision, returnRaw = false){ - let converted; - precision = Math.pow(10, precision); - const convData = splitInput(input); - // Convert everything to decimal degrees first - switch (inFormat) { - case "Degrees Minutes Seconds": - if (convData.length < 3) { - throw "Invalid co-ordinates format."; - } - converted = convDMSToDD(convData[0], convData[1], convData[2], precision); - break; - case "Degrees Decimal Minutes": - if (convData.length < 2) { - throw "Invalid co-ordinates format."; - } - converted = convDDMToDD(convData[0], convData[1], precision); - break; - case "Decimal Degrees": - if (convData.length < 1) { - throw "Invalid co-ordinates format."; - } - converted = convDDToDD(convData[0], precision); - break; - default: - throw "Unknown input format selection."; - } - - // Convert from decimal degrees to the output format - switch (outFormat) { - case "Decimal Degrees": - break; - case "Degrees Minutes Seconds": - converted = convDDToDMS(converted.degrees); - break; - case "Degrees Decimal Minutes": - converted = convDDToDDM(converted.degrees, precision); - break; - default: - throw "Unknown output format selection."; - } - if (returnRaw) { - return converted; - } else { - return converted.string; - } -} - -/** - * Split up the input using a space, and sanitise the result + * Split up the input using a space or degrees signs, and sanitise the result * @param {string} input - The input data to be split * @returns {number[]} An array of the different items in the string, stored as floats */ function splitInput (input){ const split = []; - input.split(" ").forEach(item => { + input.split(/[°′″'"\s]/).forEach(item => { // Remove any character that isn't a digit item = item.replace(/[^0-9.-]/g, ""); if (item.length > 0){ - split.push(parseFloat(item, 10)); + split.push(parseFloat(item)); } }); return split; @@ -245,47 +337,153 @@ function convDDToDDM (decDegrees, precision) { } /** - * + * Finds and returns the compass directions in an input string + * @param {string} input - The input co-ordinates containing the direction + * @param {string} delim - The delimiter separating latitide and longitude + * @returns {string[]} String array containing the latitude and longitude directions + */ +export function findDirs(input, delim) { + const upperInput = input.toUpperCase(); + const dirExp = new RegExp(/[NESW]/g); + + const dirs = upperInput.match(dirExp); + + if (dirExp.test(upperInput)) { + // If there's actually compass directions in the string + if (dirs.length <= 2 && dirs.length >= 1) { + if (dirs.length === 2) { + return [dirs[0], dirs[1]]; + } else { + return [dirs[0], ""]; + } + } + } + // Nothing was returned, so guess the directions + let lat = upperInput, + long, + latDir = "", + longDir = ""; + if (!delim.includes("Direction")) { + if (upperInput.includes(delim)) { + const split = upperInput.split(delim); + if (split.length > 1) { + if (split[0] === "") { + lat = split[1]; + } else { + lat = split[0]; + } + if (split.length > 2) { + if (split[2] !== "") { + long = split[2]; + } + } + } + } + } else { + const split = upperInput.split(dirExp); + if (split.length > 1) { + if (split[0] === "") { + lat = split[1]; + } else { + lat = split[0]; + } + if (split.length > 2) { + if (split[2] !== "") { + long = split[2]; + } + } + } + } + if (lat) { + lat = parseFloat(lat); + if (lat < 0) { + latDir = "S"; + } else { + latDir = "N"; + } + } + if (long) { + long = parseFloat(long); + if (long < 0) { + longDir = "W"; + } else { + longDir = "E"; + } + } + + return [latDir, longDir]; +} + +/** + * Detects the co-ordinate format of the input data * @param {string} input - The input data whose format we need to detect * @param {string} delim - The delimiter separating the data in input * @returns {string} The input format */ export function findFormat (input, delim) { - input = input.trim(); let testData; - if (delim.includes("Direction")) { + const mgrsPattern = new RegExp(/^[0-9]{2}\s?[C-HJ-NP-X]{1}\s?[A-HJ-NP-Z][A-HJ-NP-V]\s?[0-9\s]+/), + osngPattern = new RegExp(/^[STNHO][A-HJ-Z][0-9]+$/), + geohashPattern = new RegExp(/^[0123456789BCDEFGHJKMNPQRSTUVWXYZ]+$/), + utmPattern = new RegExp(/^[0-9]{2}\s?[C-HJ-NP-X]\s[0-9\.]+\s?[0-9\.]+$/), + degPattern = new RegExp(/[°'"]/g); + input = input.trim(); + if (delim !== null && delim.includes("Direction")) { const split = input.split(/[NnEeSsWw]/); - if (split.length > 0) { + if (split.length > 1) { if (split[0] === "") { - // Direction Preceding testData = split[1]; } else { - // Direction Following testData = split[0]; } } - } else if (delim !== "") { - const split = input.split(delim); - if (!input.includes(delim)) { + } else if (delim !== null && delim !== "") { + if (input.includes(delim)) { + const split = input.split(delim); + if (split.length > 1) { + if (split[0] === "") { + testData = split[1]; + } else { + testData = split[0]; + } + } + } else { testData = input; } - if (split.length > 0) { - if (split[0] !== "") { - testData = split[0]; - } else if (split.length > 1) { - testData = split[1]; - } - } } // Test MGRS and Geohash - if (input.split(" ").length <= 1) { - const filteredInput = input.replace(/[^A-Za-z0-9]/, "").toUpperCase(); - const mgrsPattern = new RegExp(/^[0-9]{2}[C-HJ-NP-X]{2}[A-Z]+/); - const geohashPattern = new RegExp(/^[0123456789BCDEFGHJKMNPQRSTUVWXYZ]+$/); - if (mgrsPattern.test(filteredInput)) { + if (!degPattern.test(input)) { + const filteredInput = input.toUpperCase(); + const isMgrs = mgrsPattern.test(filteredInput); + const isOsng = osngPattern.test(filteredInput); + const isGeohash = geohashPattern.test(filteredInput); + const isUtm = utmPattern.test(filteredInput); + if (isMgrs && (isOsng || isGeohash)) { + if (filteredInput.includes("I")) { + // Only MGRS can have an i! + return "Military Grid Reference System"; + } + } + if (isUtm) { + return "Universal Transverse Mercator"; + } + if (isOsng && isGeohash) { + // Geohash doesn't have A, L or O, but OSNG does. + const testExp = new RegExp(/[ALO]/g); + if (testExp.test(filteredInput)) { + return "Ordnance Survey National Grid"; + } else { + return "Geohash"; + } + } + if (isMgrs) { return "Military Grid Reference System"; - } else if (geohashPattern.test(filteredInput)) { + } + if (isOsng) { + return "Ordnance Survey National Grid"; + } + if (isGeohash) { return "Geohash"; } } @@ -312,7 +510,7 @@ export function findFormat (input, delim) { */ export function findDelim (input) { input = input.trim(); - const delims = [",", ";", ":"]; + const delims = [",", ";", ":", " "]; const testDir = input.match(/[NnEeSsWw]/g); if (testDir !== null && testDir.length > 0 && testDir.length < 3) { // Possibly contains a direction @@ -340,3 +538,19 @@ export function findDelim (input) { } return null; } + +/** + * Gets the real string for a delimiter name. + * @param {string} delim The delimiter to be matched + * @returns {string} + */ +export function realDelim (delim) { + return { + "Auto": "Auto", + "Space": " ", + "\\n": "\n", + "Comma": ",", + "Semi-colon": ";", + "Colon": ":" + }[delim]; +} diff --git a/src/core/operations/ConvertCoordinateFormat.mjs b/src/core/operations/ConvertCoordinateFormat.mjs index afc95982..770920f4 100644 --- a/src/core/operations/ConvertCoordinateFormat.mjs +++ b/src/core/operations/ConvertCoordinateFormat.mjs @@ -5,9 +5,7 @@ */ import Operation from "../Operation"; -import OperationError from "../errors/OperationError"; -import {FORMATS, STRING_FORMATS, convertCoordinates, convertSingleCoordinate, findDelim, findFormat} from "../lib/ConvertCoordinates"; -import Utils from "../Utils"; +import {FORMATS, convertCoordinates} from "../lib/ConvertCoordinates"; /** * Convert co-ordinate format operation @@ -22,7 +20,7 @@ class ConvertCoordinateFormat extends Operation { this.name = "Convert co-ordinate format"; this.module = "Hashing"; - this.description = "Convert geographical coordinates between different formats.

Supported formats:
  • Degrees Minutes Seconds (DMS)
  • Degrees Decimal Minutes (DDM)
  • Decimal Degrees (DD)
  • Geohash
  • Military Grid Reference System (MGRS)
  • Ordnance Survey National Grid (OSNG)
"; + this.description = "Convert geographical coordinates between different formats.

Supported formats:
  • Degrees Minutes Seconds (DMS)
  • Degrees Decimal Minutes (DDM)
  • Decimal Degrees (DD)
  • Geohash
  • Military Grid Reference System (MGRS)
  • Ordnance Survey National Grid (OSNG)
  • Universal Transverse Mercator (UTM)

The operation can try to detect the input co-ordinate format and delimiter automatically, but this may not always work correctly."; this.infoURL = "https://wikipedia.org/wiki/Geographic_coordinate_conversion"; this.inputType = "string"; this.outputType = "string"; @@ -39,6 +37,7 @@ class ConvertCoordinateFormat extends Operation { "Auto", "Direction Preceding", "Direction Following", + "Space", "\\n", "Comma", "Semi-colon", @@ -55,14 +54,21 @@ class ConvertCoordinateFormat extends Operation { "type": "option", "value": [ "Space", - "Direction Preceding", - "Direction Following", "\\n", "Comma", "Semi-colon", "Colon" ] }, + { + "name": "Include Compass Directions", + "type": "option", + "value": [ + "None", + "Before", + "After" + ] + }, { "name": "Precision", "type": "number", @@ -77,148 +83,8 @@ class ConvertCoordinateFormat extends Operation { * @returns {string} */ run(input, args) { - const outFormat = args[2], - outDelim = args[3], - precision = args[4]; - let inFormat = args[0], - inDelim = args[1], - inLat, - inLong, - outLat, - outLong, - latDir = "", - longDir = "", - outSeparator = " "; - - // Autodetect input delimiter - if (inDelim === "Auto") { - inDelim = findDelim(input); - if (inDelim === null) { - inDelim = ""; - } - } else if (!inDelim.includes("Direction")) { - // Get the actual delimiter from the regex - inDelim = String(Utils.regexRep(inDelim)).slice(1, 2); - } - if (inFormat === "Auto") { - inFormat = findFormat(input, inDelim); - if (inFormat === null) { - throw new OperationError("Could not automatically detect the input format."); - } - } - - if (inDelim === "" && (!STRING_FORMATS.includes(inFormat))) { - throw new OperationError("Could not automatically detect the input delimiter."); - } - - // Prepare input data - if (STRING_FORMATS.includes(inFormat)) { - // Geohash only has one value, so just use the input - // Replace anything that isn't a valid character in Geohash / MGRS / OSNG - inLat = input.replace(/[^A-Za-z0-9]/, ""); - } else if (inDelim === "Direction Preceding") { - // Split on the compass directions - const splitInput = input.split(/[NnEeSsWw]/); - const dir = input.match(/[NnEeSsWw]/g); - if (splitInput.length > 1) { - inLat = splitInput[1]; - if (dir !== null) { - latDir = dir[0]; - } - if (splitInput.length > 2) { - inLong = splitInput[2]; - if (dir !== null && dir.length > 1) { - longDir = dir[1]; - } - } - } - } else if (inDelim === "Direction Following") { - // Split on the compass directions - const splitInput = input.split(/[NnEeSsWw]/); - if (splitInput.length >= 1) { - inLat = splitInput[0]; - if (splitInput.length >= 2) { - inLong = splitInput[1]; - } - } - } else { - // Split on the delimiter - const splitInput = input.split(inDelim); - if (splitInput.length > 0) { - inLat = splitInput[0]; - if (splitInput.length >= 2) { - inLong = splitInput[1]; - } - } - } - - if (!STRING_FORMATS.includes(inFormat) && outDelim.includes("Direction")) { - // Match on compass directions, and store the first 2 matches for the output - const dir = input.match(/[NnEeSsWw]/g); - if (dir !== null) { - latDir = dir[0]; - if (dir.length > 1) { - longDir = dir[1]; - } - } - } else if (outDelim === "\\n") { - outSeparator = "\n"; - } else if (outDelim === "Space") { - outSeparator = " "; - } else if (!outDelim.includes("Direction")) { - // Cut out the regex syntax (/) from the delimiter - outSeparator = String(Utils.regexRep(outDelim)).slice(1, 2); - } - - // Convert the co-ordinates - if (inLat !== undefined) { - if (inLong === undefined) { - if (!STRING_FORMATS.includes(inFormat)) { - if (STRING_FORMATS.includes(outFormat)){ - throw new OperationError(`${outFormat} needs both a latitude and a longitude to be calculated`); - } - } - if (STRING_FORMATS.includes(inFormat)) { - // Geohash conversion is in convertCoordinates despite needing - // only one input as it needs to output two values - inLat = inLat.replace(/[^A-Za-z0-9]/g, ""); - [outLat, outLong] = convertCoordinates(inLat, inLat, inFormat, outFormat, precision); - } else { - outLat = convertSingleCoordinate(inLat, inFormat, outFormat, precision); - } - } else { - [outLat, outLong] = convertCoordinates(inLat, inLong, inFormat, outFormat, precision); - } - } else { - throw new OperationError("No co-ordinates were detected in the input."); - } - - // Output conversion results if successful - if (outLat !== undefined) { - let output = ""; - if (outDelim === "Direction Preceding" && !STRING_FORMATS.includes(outFormat)) { - output += latDir += " "; - } - output += outLat; - if (outDelim === "Direction Following" && !STRING_FORMATS.includes(outFormat)) { - output += " " + latDir; - } - output += outSeparator; - - if (outLong !== undefined && !STRING_FORMATS.includes(outFormat)) { - if (outDelim === "Direction Preceding") { - output += longDir + " "; - } - output += outLong; - if (outDelim === "Direction Following") { - output += " " + longDir; - } - output += outSeparator; - } - return output; - } else { - throw new OperationError("Co-ordinate conversion failed."); - } + const [inFormat, inDelim, outFormat, outDelim, incDirection, precision] = args; + return convertCoordinates(input, inFormat, inDelim, outFormat, outDelim, incDirection, precision); } } From 0602f457cef97046f03ef8dc81156de2a18ed613 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 15 Jan 2019 16:24:29 +0000 Subject: [PATCH 41/52] Added initial tests & counts support --- src/core/operations/YARARules.mjs | 13 ++++++++++--- tests/operations/index.mjs | 1 + tests/operations/tests/YARA.mjs | 24 ++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 tests/operations/tests/YARA.mjs diff --git a/src/core/operations/YARARules.mjs b/src/core/operations/YARARules.mjs index 58b3576c..c453d6b8 100644 --- a/src/core/operations/YARARules.mjs +++ b/src/core/operations/YARARules.mjs @@ -49,6 +49,12 @@ class YARARules extends Operation { type: "boolean", hint: "Show the metadata of each rule", value: false + }, + { + name: "Show counts", + type: "boolean", + hint: "Show the number of matches per rule", + value: true } ]; } @@ -61,7 +67,7 @@ class YARARules extends Operation { run(input, args) { if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Instantiating YARA."); - const [rules, showStrings, showLengths, showMeta] = args; + const [rules, showStrings, showLengths, showMeta, showCounts] = args; return new Promise((resolve, reject) => { Yara().then(yara => { if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Converting data for YARA."); @@ -97,10 +103,11 @@ class YARARules extends Operation { } meta = meta.slice(0, -2) + "]"; } + const countString = showCounts ? `${matches.size()} time${matches.size() > 1 ? "s" : ""}` : ""; if (matches.size() === 0 || !(showStrings || showLengths)) { - matchString += `Input matches rule "${rule.ruleName}"${meta}.\n`; + matchString += `Input matches rule "${rule.ruleName}"${meta}${countString.length > 0 ? ` ${countString}`: ""}.\n`; } else { - matchString += `Rule "${rule.ruleName}"${meta} matches:\n`; + matchString += `Rule "${rule.ruleName}"${meta} matches (${countString}):\n`; for (let j = 0; j < matches.size(); j++) { const match = matches.get(j); if (showStrings || showLengths) { diff --git a/tests/operations/index.mjs b/tests/operations/index.mjs index da9d41be..48bd08a8 100644 --- a/tests/operations/index.mjs +++ b/tests/operations/index.mjs @@ -82,6 +82,7 @@ import "./tests/TranslateDateTimeFormat"; import "./tests/Magic"; import "./tests/ParseTLV"; import "./tests/Media"; +import "./tests/YARA.mjs"; // Cannot test operations that use the File type yet //import "./tests/SplitColourChannels"; diff --git a/tests/operations/tests/YARA.mjs b/tests/operations/tests/YARA.mjs new file mode 100644 index 00000000..e3c28ef1 --- /dev/null +++ b/tests/operations/tests/YARA.mjs @@ -0,0 +1,24 @@ +/** + * YARA Rules tests. + * + * @author Matt C [matt@artemisbot.uk] + * + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ +import TestRegister from "../TestRegister"; + +TestRegister.addTests([ + { + name: "YARA Match: simple foobar", + input: "foobar foobar bar foo foobar", + expectedOutput: "Rule \"foo\" matches (4 times):\nPos 0, length 3, identifier $re1, data: \"foo\"\nPos 7, length 3, identifier $re1, data: \"foo\"\nPos 18, length 3, identifier $re1, data: \"foo\"\nPos 22, length 3, identifier $re1, data: \"foo\"\nRule \"bar\" matches (4 times):\nPos 3, length 3, identifier $re1, data: \"bar\"\nPos 10, length 3, identifier $re1, data: \"bar\"\nPos 14, length 3, identifier $re1, data: \"bar\"\nPos 25, length 3, identifier $re1, data: \"bar\"\n", + recipeConfig: [ + { + "op": "YARA Rules", + "args": ["rule foo {strings: $re1 = /foo/ condition: $re1} rule bar {strings: $re1 = /bar/ condition: $re1}", true, true, true, true], + } + ], + }, +]); + From fcc39a03973276b96aaac22b6c8c39c217f05664 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 15 Jan 2019 23:42:05 +0000 Subject: [PATCH 42/52] Added File upload support to textarea --- src/web/Manager.mjs | 3 ++ src/web/RecipeWaiter.mjs | 63 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/src/web/Manager.mjs b/src/web/Manager.mjs index d33616a4..3f4af771 100755 --- a/src/web/Manager.mjs +++ b/src/web/Manager.mjs @@ -137,6 +137,9 @@ class Manager { this.addDynamicListener("#rec-list li.operation > div", "dblclick", this.recipe.operationChildDblclick, this.recipe); this.addDynamicListener("#rec-list .dropdown-menu.toggle-dropdown a", "click", this.recipe.dropdownToggleClick, this.recipe); this.addDynamicListener("#rec-list", "operationremove", this.recipe.opRemove.bind(this.recipe)); + this.addDynamicListener("textarea.arg", "dragover", this.recipe.textArgDragover, this.recipe); + this.addDynamicListener("textarea.arg", "dragleave", this.recipe.textArgDragLeave, this.recipe); + this.addDynamicListener("textarea.arg", "drop", this.recipe.textArgDrop, this.recipe); // Input this.addMultiEventListener("#input-text", "keyup", this.input.inputChange, this.input); diff --git a/src/web/RecipeWaiter.mjs b/src/web/RecipeWaiter.mjs index b913fede..a7a9e2e3 100755 --- a/src/web/RecipeWaiter.mjs +++ b/src/web/RecipeWaiter.mjs @@ -453,6 +453,69 @@ class RecipeWaiter { window.dispatchEvent(this.manager.statechange); } + /** + * Handler for text argument dragover events. + * Gives the user a visual cue to show that items can be dropped here. + * + * @param {event} e + */ + textArgDragover (e) { + // This will be set if we're dragging an operation + if (e.dataTransfer.effectAllowed === "move") + return false; + + e.stopPropagation(); + e.preventDefault(); + e.target.closest("textarea.arg").classList.add("dropping-file"); + } + + /** + * Handler for text argument dragleave events. + * Removes the visual cue. + * + * @param {event} e + */ + textArgDragLeave (e) { + e.stopPropagation(); + e.preventDefault(); + e.target.classList.remove("dropping-file"); + } + + /** + * Handler for text argument drop events. + * Loads the dragged data into the argument textarea. + * + * @param {event} e + */ + textArgDrop(e) { + // This will be set if we're dragging an operation + if (e.dataTransfer.effectAllowed === "move") + return false; + + e.stopPropagation(); + e.preventDefault(); + const targ = e.target; + const file = e.dataTransfer.files[0]; + const text = e.dataTransfer.getData("Text"); + + targ.classList.remove("dropping-file"); + + if (text) { + targ.value = text; + return; + } + + if (file) { + const reader = new FileReader(); + const self = this; + reader.onload = function (e) { + targ.value = e.target.result; + self.ingChange(); + }; + reader.readAsText(file); + } + } + /** * Sets register values. From 3dfaaf4c2501b236ec5fddfb38df670a188e81f3 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 15 Jan 2019 23:45:40 +0000 Subject: [PATCH 43/52] Update libyara for test pass --- package-lock.json | 6 +++--- package.json | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6ba03db3..c7706dcb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7756,9 +7756,9 @@ "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA=" }, "libyara-wasm": { - "version": "0.0.9", - "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.9.tgz", - "integrity": "sha512-GTR8GD7q2xCwRzQpoSVh31KPbH145tCKaU3DAnQMpkf4aKqjnVpQlq6gUjyDhvw8RPZ2tamyQad5hg42KsVvRw==" + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.10.tgz", + "integrity": "sha512-Hn3u8wb2s6YyH8LYAVB59S05aj4vRDyLe6R3I7LvW0L2Atsc3WfaAfR1i0PgdNXYznE36FCm22pK+xLJ6CJozA==" }, "livereload-js": { "version": "2.4.0", diff --git a/package.json b/package.json index a18331d8..33c8dccd 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,7 @@ "jsqr": "^1.1.1", "jsrsasign": "8.0.12", "kbpgp": "^2.0.82", - "libyara-wasm": "0.0.9", + "libyara-wasm": "0.0.10", "lodash": "^4.17.11", "loglevel": "^1.6.1", "loglevel-message-prefix": "^3.0.0", From d1961ca3fa3b6ff37d9333d6e07e0ab580d89b68 Mon Sep 17 00:00:00 2001 From: Matt Date: Wed, 16 Jan 2019 01:15:51 +0000 Subject: [PATCH 44/52] Marginally reduced size of libyara-wasm --- package-lock.json | 60 ++++++++++++++++++++++------------------------- package.json | 2 +- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/package-lock.json b/package-lock.json index 08086ee7..932da37f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1497,7 +1497,7 @@ }, "ansi-escapes": { "version": "3.1.0", - "resolved": "http://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", "integrity": "sha512-UgAb8H9D41AQnu/PbWlCofQVcnV4Gs2bBJi9eZPxfU/hgglFh3SMDMENRIqdr7H6XFnXdoknctFByVsCOotTVw==", "dev": true }, @@ -1615,7 +1615,7 @@ }, "array-equal": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=", "dev": true }, @@ -1700,7 +1700,7 @@ }, "util": { "version": "0.10.3", - "resolved": "http://registry.npmjs.org/util/-/util-0.10.3.tgz", + "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", "dev": true, "requires": { @@ -1848,7 +1848,7 @@ }, "axios": { "version": "0.18.0", - "resolved": "http://registry.npmjs.org/axios/-/axios-0.18.0.tgz", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz", "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=", "dev": true, "requires": { @@ -2286,7 +2286,7 @@ }, "browserify-aes": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", "dev": true, "requires": { @@ -2323,7 +2323,7 @@ }, "browserify-rsa": { "version": "4.0.1", - "resolved": "http://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", + "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=", "dev": true, "requires": { @@ -2388,7 +2388,7 @@ }, "buffer": { "version": "4.9.1", - "resolved": "http://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", "dev": true, "requires": { @@ -2551,7 +2551,7 @@ }, "camelcase-keys": { "version": "2.1.0", - "resolved": "http://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=", "dev": true, "requires": { @@ -2600,7 +2600,7 @@ }, "chalk": { "version": "1.1.3", - "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", "requires": { "ansi-styles": "^2.2.1", @@ -3133,7 +3133,7 @@ }, "create-hash": { "version": "1.2.0", - "resolved": "http://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", + "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", "dev": true, "requires": { @@ -3146,7 +3146,7 @@ }, "create-hmac": { "version": "1.1.7", - "resolved": "http://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", + "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", "dev": true, "requires": { @@ -4794,7 +4794,7 @@ }, "finalhandler": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", "dev": true, "requires": { @@ -5098,14 +5098,12 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5125,8 +5123,7 @@ "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "console-control-strings": { "version": "1.1.0", @@ -5274,7 +5271,6 @@ "version": "3.0.4", "bundled": true, "dev": true, - "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5985,7 +5981,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -6129,7 +6125,7 @@ "dependencies": { "colors": { "version": "1.1.2", - "resolved": "http://registry.npmjs.org/colors/-/colors-1.1.2.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz", "integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM=", "dev": true } @@ -6193,7 +6189,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -7760,9 +7756,9 @@ "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA=" }, "libyara-wasm": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.10.tgz", - "integrity": "sha512-Hn3u8wb2s6YyH8LYAVB59S05aj4vRDyLe6R3I7LvW0L2Atsc3WfaAfR1i0PgdNXYznE36FCm22pK+xLJ6CJozA==" + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.11.tgz", + "integrity": "sha512-rglapPFo0IHPNksWYQXI8oqftXYj5mOGOf4BXtbSySVRX71pro4BehNjJ5qEpjYx+roGvNkcAD9zCsitA08sxw==" }, "livereload-js": { "version": "2.4.0", @@ -8480,7 +8476,7 @@ "dependencies": { "commander": { "version": "2.15.1", - "resolved": "http://registry.npmjs.org/commander/-/commander-2.15.1.tgz", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz", "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==", "dev": true, "optional": true @@ -8931,7 +8927,7 @@ "dependencies": { "colors": { "version": "0.5.1", - "resolved": "http://registry.npmjs.org/colors/-/colors-0.5.1.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-0.5.1.tgz", "integrity": "sha1-fQAj6usVTo7p/Oddy5I9DtFmd3Q=" }, "underscore": { @@ -10161,13 +10157,13 @@ "dependencies": { "async": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=", "dev": true }, "winston": { "version": "2.1.1", - "resolved": "http://registry.npmjs.org/winston/-/winston-2.1.1.tgz", + "resolved": "https://registry.npmjs.org/winston/-/winston-2.1.1.tgz", "integrity": "sha1-PJNJ0ZYgf9G9/51LxD73JRDjoS4=", "dev": true, "requires": { @@ -10182,7 +10178,7 @@ "dependencies": { "colors": { "version": "1.0.3", - "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true }, @@ -12947,7 +12943,7 @@ "dependencies": { "async": { "version": "0.9.2", - "resolved": "http://registry.npmjs.org/async/-/async-0.9.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=", "dev": true }, @@ -13675,14 +13671,14 @@ "dependencies": { "async": { "version": "1.0.0", - "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=", "dev": true, "optional": true }, "colors": { "version": "1.0.3", - "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true, "optional": true diff --git a/package.json b/package.json index 4dd5b07b..dd71321f 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,7 @@ "jsqr": "^1.1.1", "jsrsasign": "8.0.12", "kbpgp": "^2.0.82", - "libyara-wasm": "0.0.10", + "libyara-wasm": "0.0.11", "lodash": "^4.17.11", "loglevel": "^1.6.1", "loglevel-message-prefix": "^3.0.0", From 69797e58cb6fc5d5622c55dbd42fab83de1ad668 Mon Sep 17 00:00:00 2001 From: j433866 Date: Wed, 16 Jan 2019 16:57:58 +0000 Subject: [PATCH 45/52] Add better error handling. Also now doesn't do anything if there's no input --- src/core/operations/ConvertCoordinateFormat.mjs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/core/operations/ConvertCoordinateFormat.mjs b/src/core/operations/ConvertCoordinateFormat.mjs index 770920f4..09e1620c 100644 --- a/src/core/operations/ConvertCoordinateFormat.mjs +++ b/src/core/operations/ConvertCoordinateFormat.mjs @@ -5,6 +5,7 @@ */ import Operation from "../Operation"; +import OperationError from "../errors/OperationError"; import {FORMATS, convertCoordinates} from "../lib/ConvertCoordinates"; /** @@ -37,7 +38,6 @@ class ConvertCoordinateFormat extends Operation { "Auto", "Direction Preceding", "Direction Following", - "Space", "\\n", "Comma", "Semi-colon", @@ -83,8 +83,17 @@ class ConvertCoordinateFormat extends Operation { * @returns {string} */ run(input, args) { - const [inFormat, inDelim, outFormat, outDelim, incDirection, precision] = args; - return convertCoordinates(input, inFormat, inDelim, outFormat, outDelim, incDirection, precision); + if (input.replace(/[\s+]/g, "") !== "") { + const [inFormat, inDelim, outFormat, outDelim, incDirection, precision] = args; + try { + const result = convertCoordinates(input, inFormat, inDelim, outFormat, outDelim, incDirection, precision); + return result; + } catch (error) { + throw new OperationError(error); + } + } else { + return input; + } } } From 439654ed7f11d7c7bf3a4364d0f4bb95dce34aee Mon Sep 17 00:00:00 2001 From: j433866 Date: Thu, 17 Jan 2019 13:49:36 +0000 Subject: [PATCH 46/52] Add tests for new co-ordinate conversion module. Removed To/From geohash tests --- test/index.mjs | 3 +- .../operations/ConvertCoordinateFormat.mjs | 211 ++++++++++++++++++ test/tests/operations/FromGeohash.mjs | 55 ----- test/tests/operations/ToGeohash.mjs | 55 ----- 4 files changed, 212 insertions(+), 112 deletions(-) create mode 100644 test/tests/operations/ConvertCoordinateFormat.mjs delete mode 100644 test/tests/operations/FromGeohash.mjs delete mode 100644 test/tests/operations/ToGeohash.mjs diff --git a/test/index.mjs b/test/index.mjs index 9c11f6ae..8d7bd798 100644 --- a/test/index.mjs +++ b/test/index.mjs @@ -44,7 +44,6 @@ import "./tests/operations/DateTime"; import "./tests/operations/ExtractEmailAddresses"; import "./tests/operations/Fork"; import "./tests/operations/FromDecimal"; -import "./tests/operations/FromGeohash"; import "./tests/operations/Hash"; import "./tests/operations/HaversineDistance"; import "./tests/operations/Hexdump"; @@ -74,10 +73,10 @@ import "./tests/operations/SetIntersection"; import "./tests/operations/SetUnion"; import "./tests/operations/StrUtils"; import "./tests/operations/SymmetricDifference"; -import "./tests/operations/ToGeohash.mjs"; import "./tests/operations/TranslateDateTimeFormat"; import "./tests/operations/Magic"; import "./tests/operations/ParseTLV"; +import "./tests/operations/ConvertCoordinateFormat"; let allTestsPassing = true; const testStatusCounts = { diff --git a/test/tests/operations/ConvertCoordinateFormat.mjs b/test/tests/operations/ConvertCoordinateFormat.mjs new file mode 100644 index 00000000..89690b97 --- /dev/null +++ b/test/tests/operations/ConvertCoordinateFormat.mjs @@ -0,0 +1,211 @@ +/** + * Convert co-ordinate format tests + * + * @author j433866 + * + * @copyright Crown Copyright 2019 + * @license Apache-2.0 + */ + +/** + * TEST CO-ORDINATES + * DD: 51.504°,-0.126°, + * DDM: 51° 30.24',-0° 7.56', + * DMS: 51° 30' 14.4",-0° 7' 33.6", + * Geohash: gcpvj0h0x, + * MGRS: 30U XC 99455 09790, + * OSNG: TQ 30163 80005, + * UTM: 30N 699456 5709791, + */ + +import TestRegister from "../../TestRegister"; + +TestRegister.addTests([ + { + name: "Co-ordinates: From Decimal Degrees to Degrees Minutes Seconds", + input: "51.504°,-0.126°,", + expectedOutput: "51° 30' 14.4\",-0° 7' 33.6\",", + recipeConfig: [ + { + op: "Convert co-ordinate format", + args: ["Decimal Degrees", "Comma", "Degrees Minutes Seconds", "Comma", "None", 1] + }, + ], + }, + { + name: "Co-ordinates: From Degrees Minutes Seconds to Decimal Degrees", + input: "51° 30' 14.4\",-0° 7' 33.6\",", + expectedOutput: "51.504°,-0.126°,", + recipeConfig: [ + { + op: "Convert co-ordinate format", + args: ["Degrees Minutes Seconds", "Comma", "Decimal Degrees", "Comma", "None", 3] + }, + ], + }, + { + name: "Co-ordinates: From Decimal Degrees to Degrees Decimal Minutes", + input: "51.504°,-0.126°,", + expectedOutput: "51° 30.24',-0° 7.56',", + recipeConfig: [ + { + op: "Convert co-ordinate format", + args: ["Decimal Degrees", "Comma", "Degrees Decimal Minutes", "Comma", "None", 2] + } + ] + }, + { + name: "Co-ordinates: From Degrees Decimal Minutes to Decimal Degrees", + input: "51° 30.24',-0° 7.56',", + expectedOutput: "51.504°,-0.126°,", + recipeConfig: [ + { + op: "Convert co-ordinate format", + args: ["Degrees Decimal Minutes", "Comma", "Decimal Degrees", "Comma", "None", 3] + } + ] + }, + { + name: "Co-ordinates: From Decimal Degrees to Decimal Degrees", + input: "51.504°,-0.126°,", + expectedOutput: "51.504°,-0.126°,", + recipeConfig: [ + { + op: "Convert co-ordinate format", + args: ["Decimal Degrees", "Comma", "Decimal Degrees", "Comma", "None", 3] + } + ] + }, + { + name: "Co-ordinates: From Decimal Degrees to Geohash", + input: "51.504°,-0.126°,", + expectedOutput: "gcpvj0h0x,", + recipeConfig: [ + { + op: "Convert co-ordinate format", + args: ["Decimal Degrees", "Comma", "Geohash", "Comma", "None", 9] + }, + ], + }, + { + name: "Co-ordinates: From Geohash to Decimal Degrees", + input: "gcpvj0h0x,", + expectedOutput: "51.504°,-0.126°,", + recipeConfig: [ + { + op: "Convert co-ordinate format", + args: ["Geohash", "Comma", "Decimal Degrees", "Comma", "None", 3] + }, + ], + }, + { + name: "Co-ordinates: From Decimal Degrees to MGRS", + input: "51.504°,-0.126°,", + expectedOutput: "30U XC 99455 09790,", + recipeConfig: [ + { + op: "Convert co-ordinate format", + args: ["Decimal Degrees", "Comma", "Military Grid Reference System", "Comma", "None", 10] + }, + ], + }, + { + name: "Co-ordinates: From MGRS to Decimal Degrees", + input: "30U XC 99455 09790,", + expectedOutput: "51.504°,-0.126°,", + recipeConfig: [ + { + op: "Convert co-ordinate format", + args: ["Military Grid Reference System", "Comma", "Decimal Degrees", "Comma", "None", 3] + } + ] + }, + { + name: "Co-ordinates: From Decimal Degrees to OSNG", + input: "51.504°,-0.126°,", + expectedOutput: "TQ 30163 80005,", + recipeConfig: [ + { + op: "Convert co-ordinate format", + args: ["Decimal Degrees", "Comma", "Ordnance Survey National Grid", "Comma", "None", 10] + }, + ], + }, + { + name: "Co-ordinates: From OSNG to Decimal Degrees", + input: "TQ 30163 80005,", + expectedOutput: "51.504°,-0.126°,", + recipeConfig: [ + { + op: "Convert co-ordinate format", + args: ["Ordnance Survey National Grid", "Comma", "Decimal Degrees", "Comma", "None", 3] + }, + ], + }, + { + name: "Co-ordinates: From Decimal Degrees to UTM", + input: "51.504°,-0.126°,", + expectedOutput: "30 N 699456 5709791,", + recipeConfig: [ + { + op: "Convert co-ordinate format", + args: ["Decimal Degrees", "Comma", "Universal Transverse Mercator", "Comma", "None", 0] + }, + ], + }, + { + name: "Co-ordinates: From UTM to Decimal Degrees", + input: "30 N 699456 5709791,", + expectedOutput: "51.504°,-0.126°,", + recipeConfig: [ + { + op: "Convert co-ordinate format", + args: ["Universal Transverse Mercator", "Comma", "Decimal Degrees", "Comma", "None", 3] + }, + ], + }, + { + name: "Co-ordinates: Directions in input, not output", + input: "N51.504°,W0.126°,", + expectedOutput: "51.504°,-0.126°,", + recipeConfig: [ + { + op: "Convert co-ordinate format", + args: ["Decimal Degrees", "Comma", "Decimal Degrees", "Comma", "None", 3] + }, + ], + }, + { + name: "Co-ordinates: Directions in input and output", + input: "N51.504°,W0.126°,", + expectedOutput: "N 51.504°,W 0.126°,", + recipeConfig: [ + { + op: "Convert co-ordinate format", + args: ["Decimal Degrees", "Comma", "Decimal Degrees", "Comma", "Before", 3] + }, + ], + }, + { + name: "Co-ordinates: Directions not in input, in output", + input: "51.504°,-0.126°,", + expectedOutput: "N 51.504°,W 0.126°,", + recipeConfig: [ + { + op: "Convert co-ordinate format", + args: ["Decimal Degrees", "Comma", "Decimal Degrees", "Comma", "Before", 3] + }, + ], + }, + { + name: "Co-ordinates: Directions not in input, in converted output", + input: "51.504°,-0.126°,", + expectedOutput: "N 51° 30' 14.4\",W 0° 7' 33.6\",", + recipeConfig: [ + { + op: "Convert co-ordinate format", + args: ["Decimal Degrees", "Comma", "Degrees Minutes Seconds", "Comma", "Before", 3] + }, + ], + } +]); diff --git a/test/tests/operations/FromGeohash.mjs b/test/tests/operations/FromGeohash.mjs deleted file mode 100644 index 2ac68c58..00000000 --- a/test/tests/operations/FromGeohash.mjs +++ /dev/null @@ -1,55 +0,0 @@ -/** - * To Geohash tests - * - * @author gchq77703 - * @copyright Crown Copyright 2018 - * @license Apache-2.0 - */ -import TestRegister from "../../TestRegister"; - -TestRegister.addTests([ - { - name: "From Geohash", - input: "ww8p1r4t8", - expectedOutput: "37.83238649368286,112.55838632583618", - recipeConfig: [ - { - op: "From Geohash", - args: [], - }, - ], - }, - { - name: "From Geohash", - input: "ww8p1r", - expectedOutput: "37.83416748046875,112.5604248046875", - recipeConfig: [ - { - op: "From Geohash", - args: [], - }, - ], - }, - { - name: "From Geohash", - input: "ww8", - expectedOutput: "37.265625,113.203125", - recipeConfig: [ - { - op: "From Geohash", - args: [], - }, - ], - }, - { - name: "From Geohash", - input: "w", - expectedOutput: "22.5,112.5", - recipeConfig: [ - { - op: "From Geohash", - args: [], - }, - ], - }, -]); diff --git a/test/tests/operations/ToGeohash.mjs b/test/tests/operations/ToGeohash.mjs deleted file mode 100644 index b50e7280..00000000 --- a/test/tests/operations/ToGeohash.mjs +++ /dev/null @@ -1,55 +0,0 @@ -/** - * To Geohash tests - * - * @author gchq77703 - * @copyright Crown Copyright 2018 - * @license Apache-2.0 - */ -import TestRegister from "../../TestRegister"; - -TestRegister.addTests([ - { - name: "To Geohash", - input: "37.8324,112.5584", - expectedOutput: "ww8p1r4t8", - recipeConfig: [ - { - op: "To Geohash", - args: [9], - }, - ], - }, - { - name: "To Geohash", - input: "37.9324,-112.2584", - expectedOutput: "9w8pv3ruj", - recipeConfig: [ - { - op: "To Geohash", - args: [9], - }, - ], - }, - { - name: "To Geohash", - input: "37.8324,112.5584", - expectedOutput: "ww8", - recipeConfig: [ - { - op: "To Geohash", - args: [3], - }, - ], - }, - { - name: "To Geohash", - input: "37.9324,-112.2584", - expectedOutput: "9w8pv3rujxy5b99", - recipeConfig: [ - { - op: "To Geohash", - args: [15], - }, - ], - }, -]); From 4bd923dc063f901d54237204834ad0b85295964b Mon Sep 17 00:00:00 2001 From: j433866 Date: Thu, 17 Jan 2019 13:53:42 +0000 Subject: [PATCH 47/52] Improved handling of negative numbers and weirder inputs. Negative numbers shouldn't make it go weird any more. Automatic detection of input formats should be more reliable. --- src/core/lib/ConvertCoordinates.mjs | 352 +++++++++++++++++----------- 1 file changed, 212 insertions(+), 140 deletions(-) diff --git a/src/core/lib/ConvertCoordinates.mjs b/src/core/lib/ConvertCoordinates.mjs index cec2439c..d4641fe6 100644 --- a/src/core/lib/ConvertCoordinates.mjs +++ b/src/core/lib/ConvertCoordinates.mjs @@ -23,8 +23,7 @@ export const FORMATS = [ ]; /** - * Formats that should be passed to Geodesy module as-is - * Spaces are still removed + * Formats that should be passed to the conversion module as-is */ const NO_CHANGE = [ "Geohash", @@ -48,40 +47,50 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli let isPair = false, split, latlon, - conv, - inLatDir, - inLongDir; + convLat, + convLon, + conv; + // Can't have a precision less than 0! + if (precision < 0) { + precision = 0; + } if (inDelim === "Auto") { + // Try to detect a delimiter in the input. inDelim = findDelim(input); + if (inDelim === null) { + throw "Unable to detect the input delimiter automatically."; + } } else { + // Convert the delimiter argument value to the actual character inDelim = realDelim(inDelim); } if (inFormat === "Auto") { + // Try to detect the format of the input data inFormat = findFormat(input, inDelim); if (inFormat === null) { throw "Unable to detect the input format automatically."; } } - if (inDelim === null && !inFormat.includes("Direction")) { - throw "Unable to detect the input delimiter automatically."; - } + // Convert the output delimiter argument to the real character outDelim = realDelim(outDelim); if (!NO_CHANGE.includes(inFormat)) { split = input.split(inDelim); + // Replace any co-ordinate symbols with spaces so we can split on them later + for (let i = 0; i < split.length; i++) { + split[i] = split[i].replace(/[°˝´'"]/g, " "); + } if (split.length > 1) { isPair = true; } } else { + // Remove any delimiters from the input input = input.replace(inDelim, ""); isPair = true; } - if (inFormat.includes("Degrees")) { - [inLatDir, inLongDir] = findDirs(input, inDelim); - } - + // Conversions from the input format into a geodesy latlon object if (inFormat === "Geohash") { const hash = geohash.decode(input.replace(/[^A-Za-z0-9]/g, "")); latlon = new geodesy.LatLonEllipsoidal(hash.latitude, hash.longitude); @@ -92,6 +101,7 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli const osng = geodesy.OsGridRef.parse(input.replace(/[^A-Za-z0-9]/g, "")); latlon = geodesy.OsGridRef.osGridToLatLon(osng); } else if (inFormat === "Universal Transverse Mercator") { + // Geodesy needs a space between the first 2 digits and the next letter if (/^[\d]{2}[A-Za-z]/.test(input)) { input = input.slice(0, 2) + " " + input.slice(2); } @@ -99,23 +109,25 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli latlon = utm.toLatLonE(); } else if (inFormat === "Degrees Minutes Seconds") { if (isPair) { - split[0] = split[0].replace(/[NnEeSsWw]/g, "").trim(); - split[1] = split[1].replace(/[NnEeSsWw]/g, "").trim(); - const splitLat = split[0].split(/[°′″'"\s]/g), - splitLong = split[1].split(/[°′″'"\s]/g); + // Split up the lat/long into degrees / minutes / seconds values + const splitLat = splitInput(split[0]), + splitLong = splitInput(split[1]); if (splitLat.length >= 3 && splitLong.length >= 3) { - const lat = convDMSToDD(parseFloat(splitLat[0]), parseFloat(splitLat[1]), parseFloat(splitLat[2]), 10); - const long = convDMSToDD(parseFloat(splitLong[0]), parseFloat(splitLong[1]), parseFloat(splitLong[2]), 10); + const lat = convDMSToDD(splitLat[0], splitLat[1], splitLat[2], 10); + const long = convDMSToDD(splitLong[0], splitLong[1], splitLong[2], 10); latlon = new geodesy.LatLonEllipsoidal(lat.degrees, long.degrees); + } else { + throw "Invalid co-ordinate format for Degrees Minutes Seconds"; } } else { - // Create a new latlon object anyway, but we can ignore the lon value - split[0] = split[0].replace(/[NnEeSsWw]/g, "").trim(); - const splitLat = split[0].split(/[°′″'"\s]/g); + // Not a pair, so only try to convert one set of co-ordinates + const splitLat = splitInput(split[0]); if (splitLat.length >= 3) { - const lat = convDMSToDD(parseFloat(splitLat[0]), parseFloat(splitLat[1]), parseFloat(splitLat[2])); + const lat = convDMSToDD(splitLat[0], splitLat[1], splitLat[2]); latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lat.degrees); + } else { + throw "Invalid co-ordinate format for Degrees Minutes Seconds"; } } } else if (inFormat === "Degrees Decimal Minutes") { @@ -125,10 +137,12 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli if (splitLat.length !== 2 || splitLong.length !== 2) { throw "Invalid co-ordinate format for Degrees Decimal Minutes."; } + // Convert to decimal degrees, and then convert to a geodesy object const lat = convDDMToDD(splitLat[0], splitLat[1], 10); const long = convDDMToDD(splitLong[0], splitLong[1], 10); latlon = new geodesy.LatLonEllipsoidal(lat.degrees, long.degrees); } else { + // Not a pair, so only try to convert one set of co-ordinates const splitLat = splitInput(input); if (splitLat.length !== 2) { throw "Invalid co-ordinate format for Degrees Decimal Minutes."; @@ -145,6 +159,7 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli } latlon = new geodesy.LatLonEllipsoidal(splitLat[0], splitLong[0]); } else { + // Not a pair, so only try to convert one set of co-ordinates const splitLat = splitInput(split[0]); if (splitLat.length !== 1) { throw "Invalid co-ordinate format for Decimal Degrees."; @@ -152,88 +167,115 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli latlon = new geodesy.LatLonEllipsoidal(splitLat[0], splitLat[0]); } } else { - throw "Invalid input co-ordinate format selected."; + throw `Unknown input format '${inFormat}'`; } // Everything is now a geodesy latlon object + // These store the latitude and longitude as decimal + if (inFormat.includes("Degrees")) { + // If the input string contains directions, we need to check if they're S or W. + // If either of the directions are, we should make the decimal value negative + const dirs = input.match(/[NnEeSsWw]/g); + if (dirs && dirs.length >= 1) { + // Make positive lat/lon values with S/W directions into negative values + if (dirs[0] === "S" || dirs[0] === "W" && latlon.lat > 0) { + latlon.lat = 0 - latlon.lat; + } + if (dirs.length >= 2) { + if (dirs[1] === "S" || dirs[1] === "W" && latlon.lon > 0) { + latlon.lon = 0 - latlon.lon; + } + } + } + } + // Try to find the compass directions of the lat and long + const [latDir, longDir] = findDirs(latlon.lat + "," + latlon.lon, ","); + // Output conversions for each output format if (outFormat === "Decimal Degrees") { - conv = latlon.toString("d", precision); - if (!isPair) { - conv = conv.split(",")[0]; - } + // We could use the built in latlon.toString(), + // but this makes adjusting the output harder + const lat = convDDToDD(latlon.lat, precision); + const lon = convDDToDD(latlon.lon, precision); + convLat = lat.string; + convLon = lon.string; } else if (outFormat === "Degrees Decimal Minutes") { - conv = latlon.toString("dm", precision); - if (!isPair) { - conv = conv.split(",")[0]; - } + const lat = convDDToDDM(latlon.lat, precision); + const lon = convDDToDDM(latlon.lon, precision); + convLat = lat.string; + convLon = lon.string; } else if (outFormat === "Degrees Minutes Seconds") { - conv = latlon.toString("dms", precision); - if (!isPair) { - conv = conv.split(",")[0]; - } + const lat = convDDToDMS(latlon.lat, precision); + const lon = convDDToDMS(latlon.lon, precision); + convLat = lat.string; + convLon = lon.string; } else if (outFormat === "Geohash") { - conv = geohash.encode(latlon.lat.toString(), latlon.lon.toString(), precision); + convLat = geohash.encode(latlon.lat, latlon.lon, precision); } else if (outFormat === "Military Grid Reference System") { const utm = latlon.toUtm(); const mgrs = utm.toMgrs(); - conv = mgrs.toString(precision); + // MGRS wants a precision that's an even number between 2 and 10 + if (precision % 2 !== 0) { + precision = precision + 1; + } + if (precision > 10) { + precision = 10; + } + convLat = mgrs.toString(precision); } else if (outFormat === "Ordnance Survey National Grid") { const osng = geodesy.OsGridRef.latLonToOsGrid(latlon); if (osng.toString() === "") { throw "Could not convert co-ordinates to OS National Grid. Are the co-ordinates in range?"; } - conv = osng.toString(precision); + // OSNG wants a precision that's an even number between 2 and 10 + if (precision % 2 !== 0) { + precision = precision + 1; + } + if (precision > 10) { + precision = 10; + } + convLat = osng.toString(precision); } else if (outFormat === "Universal Transverse Mercator") { const utm = latlon.toUtm(); - conv = utm.toString(precision); + convLat = utm.toString(precision); } - if (conv === undefined) { + if (convLat === undefined) { throw "Error converting co-ordinates."; } + if (outFormat.includes("Degrees")) { - let [latDir, longDir] = findDirs(conv, outDelim); - if (inLatDir !== undefined) { - latDir = inLatDir; + // Format DD/DDM/DMS for output + // If we're outputting a compass direction, remove the negative sign + if (latDir === "S" && includeDir !== "None") { + convLat = convLat.replace("-", ""); } - if (inLongDir !== undefined) { - longDir = inLongDir; + if (longDir === "W" && includeDir !== "None") { + convLon = convLon.replace("-", ""); } - // DMS/DDM/DD - conv = conv.replace(", ", outDelim); - // Remove any directions from the current string, - // so we can put them where we want them - conv = conv.replace(/[NnEeSsWw]/g, ""); - if (includeDir !== "None") { - let outConv = ""; - if (!isPair) { - if (includeDir === "Before") { - outConv += latDir + " " + conv; - } else { - outConv += conv + " " + latDir; - } - } else { - const splitConv = conv.split(outDelim); - if (splitConv.length === 2) { - if (includeDir === "Before") { - outConv += latDir + " "; - } - outConv += splitConv[0]; - if (includeDir === "After") { - outConv += " " + latDir; - } - outConv += outDelim; - if (includeDir === "Before") { - outConv += longDir + " "; - } - outConv += splitConv[1]; - if (includeDir === "After") { - outConv += " " + longDir; - } - } + + let outConv = ""; + if (includeDir === "Before") { + outConv += latDir + " "; + } + + outConv += convLat; + if (includeDir === "After") { + outConv += " " + latDir; + } + outConv += outDelim; + if (isPair) { + if (includeDir === "Before") { + outConv += longDir + " "; } - conv = outConv; + outConv += convLon; + if (includeDir === "After") { + outConv += " " + longDir; + } + outConv += outDelim; } + conv = outConv; + } else { + conv = convLat + outDelim; } return conv; @@ -247,10 +289,11 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli function splitInput (input){ const split = []; - input.split(/[°′″'"\s]/).forEach(item => { - // Remove any character that isn't a digit + input.split(/\s+/).forEach(item => { + // Remove any character that isn't a digit, decimal point or negative sign item = item.replace(/[^0-9.-]/g, ""); if (item.length > 0){ + // Turn the item into a float split.push(parseFloat(item)); } }); @@ -266,10 +309,17 @@ function splitInput (input){ * @returns {{string: string, degrees: number}} An object containing the raw converted value (obj.degrees), and a formatted string version (obj.string) */ function convDMSToDD (degrees, minutes, seconds, precision){ - const converted = new Object(); - converted.degrees = degrees + (minutes / 60) + (seconds / 3600); - converted.string = (Math.round(converted.degrees * precision) / precision) + "°"; - return converted; + const absDegrees = Math.abs(degrees); + let conv = absDegrees + (minutes / 60) + (seconds / 3600); + let outString = round(conv, precision) + "°"; + if (isNegativeZero(degrees) || degrees < 0) { + conv = -conv; + outString = "-" + outString; + } + return { + "degrees": conv, + "string": outString + }; } /** @@ -280,10 +330,17 @@ function convDMSToDD (degrees, minutes, seconds, precision){ * @returns {{string: string, degrees: number}} An object containing the raw converted value (obj.degrees), and a formatted string version (obj.string) */ function convDDMToDD (degrees, minutes, precision) { - const converted = new Object(); - converted.degrees = degrees + minutes / 60; - converted.string = ((Math.round(converted.degrees * precision) / precision) + "°"); - return converted; + const absDegrees = Math.abs(degrees); + let conv = absDegrees + minutes / 60; + let outString = round(conv, precision) + "°"; + if (isNegativeZero(degrees) || degrees < 0) { + conv = -conv; + outString = "-" + outString; + } + return { + "degrees": conv, + "string": outString + }; } /** @@ -294,28 +351,34 @@ function convDDMToDD (degrees, minutes, precision) { * @returns {{string: string, degrees: number}} An object containing the raw converted value (obj.degrees), and a formatted string version (obj.string) */ function convDDToDD (degrees, precision) { - const converted = new Object(); - converted.degrees = degrees; - converted.string = Math.round(converted.degrees * precision) / precision + "°"; - return converted; + return { + "degrees": degrees, + "string": round(degrees, precision) + "°" + }; } /** * Convert Decimal Degrees to Degrees Minutes Seconds * @param {number} decDegrees - The input data to be converted + * @param {number} precision - The precision which the result should be rounded to * @returns {{string: string, degrees: number, minutes: number, seconds: number}} An object containing the raw converted value as separate numbers (.degrees, .minutes, .seconds), and a formatted string version (obj.string) */ -function convDDToDMS (decDegrees) { - const degrees = Math.floor(decDegrees); - const minutes = Math.floor(60 * (decDegrees - degrees)); - const seconds = Math.round(3600 * (decDegrees - degrees) - 60 * minutes); - - const converted = new Object(); - converted.degrees = degrees; - converted.minutes = minutes; - converted.seconds = seconds; - converted.string = degrees + "° " + minutes + "' " + seconds + "\""; - return converted; +function convDDToDMS (decDegrees, precision) { + const absDegrees = Math.abs(decDegrees); + let degrees = Math.floor(absDegrees); + const minutes = Math.floor(60 * (absDegrees - degrees)), + seconds = round(3600 * (absDegrees - degrees) - 60 * minutes, precision); + let outString = degrees + "° " + minutes + "' " + seconds + "\""; + if (isNegativeZero(decDegrees) || decDegrees < 0) { + degrees = -degrees; + outString = "-" + outString; + } + return { + "degrees": degrees, + "minutes": minutes, + "seconds": seconds, + "string": outString + }; } /** @@ -325,15 +388,21 @@ function convDDToDMS (decDegrees) { * @returns {{string: string, degrees: number, minutes: number}} An object containing the raw converted value as separate numbers (.degrees, .minutes), and a formatted string version (obj.string) */ function convDDToDDM (decDegrees, precision) { - const degrees = Math.floor(decDegrees); - const minutes = decDegrees - degrees; - const decMinutes = Math.round((minutes * 60) * precision) / precision; + const absDegrees = Math.abs(decDegrees); + let degrees = Math.floor(absDegrees); + const minutes = absDegrees - degrees, + decMinutes = round(minutes * 60, precision); + let outString = degrees + "° " + decMinutes + "'"; + if (decDegrees < 0 || isNegativeZero(decDegrees)) { + degrees = -degrees; + outString = "-" + outString; + } - const converted = new Object(); - converted.degrees = degrees; - converted.minutes = decMinutes; - converted.string = degrees + "° " + decMinutes + "'"; - return converted; + return { + "degrees": degrees, + "minutes": decMinutes, + "string": outString, + }; } /** @@ -348,8 +417,9 @@ export function findDirs(input, delim) { const dirs = upperInput.match(dirExp); - if (dirExp.test(upperInput)) { - // If there's actually compass directions in the string + if (dirs) { + // If there's actually compass directions + // in the input, use these to work out the direction if (dirs.length <= 2 && dirs.length >= 1) { if (dirs.length === 2) { return [dirs[0], dirs[1]]; @@ -366,15 +436,13 @@ export function findDirs(input, delim) { if (!delim.includes("Direction")) { if (upperInput.includes(delim)) { const split = upperInput.split(delim); - if (split.length > 1) { - if (split[0] === "") { - lat = split[1]; - } else { + if (split.length >= 1) { + if (split[0] !== "") { lat = split[0]; } - if (split.length > 2) { - if (split[2] !== "") { - long = split[2]; + if (split.length >= 2) { + if (split[1] !== "") { + long = split[1]; } } } @@ -423,9 +491,9 @@ export function findDirs(input, delim) { export function findFormat (input, delim) { let testData; const mgrsPattern = new RegExp(/^[0-9]{2}\s?[C-HJ-NP-X]{1}\s?[A-HJ-NP-Z][A-HJ-NP-V]\s?[0-9\s]+/), - osngPattern = new RegExp(/^[STNHO][A-HJ-Z][0-9]+$/), + osngPattern = new RegExp(/^[A-HJ-Z]{2}\s+[0-9\s]+$/), geohashPattern = new RegExp(/^[0123456789BCDEFGHJKMNPQRSTUVWXYZ]+$/), - utmPattern = new RegExp(/^[0-9]{2}\s?[C-HJ-NP-X]\s[0-9\.]+\s?[0-9\.]+$/), + utmPattern = new RegExp(/^[0-9]{2}\s?[C-HJ-NP-X]\s[0-9.]+\s?[0-9.]+$/), degPattern = new RegExp(/[°'"]/g); input = input.trim(); if (delim !== null && delim.includes("Direction")) { @@ -452,31 +520,16 @@ export function findFormat (input, delim) { } } - // Test MGRS and Geohash + // Test non-degrees formats if (!degPattern.test(input)) { - const filteredInput = input.toUpperCase(); + const filteredInput = input.toUpperCase().replace(delim, ""); const isMgrs = mgrsPattern.test(filteredInput); const isOsng = osngPattern.test(filteredInput); const isGeohash = geohashPattern.test(filteredInput); const isUtm = utmPattern.test(filteredInput); - if (isMgrs && (isOsng || isGeohash)) { - if (filteredInput.includes("I")) { - // Only MGRS can have an i! - return "Military Grid Reference System"; - } - } if (isUtm) { return "Universal Transverse Mercator"; } - if (isOsng && isGeohash) { - // Geohash doesn't have A, L or O, but OSNG does. - const testExp = new RegExp(/[ALO]/g); - if (testExp.test(filteredInput)) { - return "Ordnance Survey National Grid"; - } else { - return "Geohash"; - } - } if (isMgrs) { return "Military Grid Reference System"; } @@ -510,7 +563,7 @@ export function findFormat (input, delim) { */ export function findDelim (input) { input = input.trim(); - const delims = [",", ";", ":", " "]; + const delims = [",", ";", ":"]; const testDir = input.match(/[NnEeSsWw]/g); if (testDir !== null && testDir.length > 0 && testDir.length < 3) { // Possibly contains a direction @@ -554,3 +607,22 @@ export function realDelim (delim) { "Colon": ":" }[delim]; } + +/** + * Returns true if a zero is negative + * @param {number} zero + */ +function isNegativeZero(zero) { + return zero === 0 && (1/zero < 0); +} + +/** + * Rounds a number to a specified number of decimal places + * @param {number} input - The number to be rounded + * @param {precision} precision - The number of decimal places the number should be rounded to + * @returns {number} + */ +function round(input, precision) { + precision = Math.pow(10, precision); + return Math.round(input * precision) / precision; +} From 445a85798b49ae6d3817d7b86baa3b5db7f4e611 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 18 Jan 2019 14:50:00 +0000 Subject: [PATCH 48/52] 8.22.1 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 00e57f64..75645253 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.22.0", + "version": "8.22.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 96053213..796e4ec4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.22.0", + "version": "8.22.1", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From ba04cac7acbb5212dbe4beb605e75c88c0e0412a Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 18 Jan 2019 15:34:56 +0000 Subject: [PATCH 49/52] Tidied up YARA operation --- CHANGELOG.md | 5 + package-lock.json | 185 ++++++++++++++++------------- src/core/operations/YARARules.mjs | 6 +- src/web/RecipeWaiter.mjs | 10 +- src/web/stylesheets/layout/_io.css | 1 + 5 files changed, 118 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71949c82..b8f11e2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master). +### [8.23.0] - 2019-01-18 +- 'YARA Rules' operatio added [@artemisbot] | [#468] + ### [8.22.0] - 2019-01-10 - 'Subsection' operation added [@j433866] | [#467] @@ -97,6 +100,7 @@ All major and minor version changes will be documented in this file. Details of +[8.23.0]: https://github.com/gchq/CyberChef/releases/tag/v8.23.0 [8.22.0]: https://github.com/gchq/CyberChef/releases/tag/v8.22.0 [8.21.0]: https://github.com/gchq/CyberChef/releases/tag/v8.21.0 [8.20.0]: https://github.com/gchq/CyberChef/releases/tag/v8.20.0 @@ -176,3 +180,4 @@ All major and minor version changes will be documented in this file. Details of [#458]: https://github.com/gchq/CyberChef/pull/458 [#461]: https://github.com/gchq/CyberChef/pull/461 [#467]: https://github.com/gchq/CyberChef/pull/467 +[#468]: https://github.com/gchq/CyberChef/pull/468 diff --git a/package-lock.json b/package-lock.json index 3d08c64e..469dfa1e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1497,7 +1497,7 @@ }, "ansi-escapes": { "version": "3.1.0", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", + "resolved": "http://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz", "integrity": "sha512-UgAb8H9D41AQnu/PbWlCofQVcnV4Gs2bBJi9eZPxfU/hgglFh3SMDMENRIqdr7H6XFnXdoknctFByVsCOotTVw==", "dev": true }, @@ -1615,7 +1615,7 @@ }, "array-equal": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz", "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=", "dev": true }, @@ -1700,7 +1700,7 @@ }, "util": { "version": "0.10.3", - "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", + "resolved": "http://registry.npmjs.org/util/-/util-0.10.3.tgz", "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", "dev": true, "requires": { @@ -1848,7 +1848,7 @@ }, "axios": { "version": "0.18.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz", + "resolved": "http://registry.npmjs.org/axios/-/axios-0.18.0.tgz", "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=", "dev": true, "requires": { @@ -2286,7 +2286,7 @@ }, "browserify-aes": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", "dev": true, "requires": { @@ -2323,7 +2323,7 @@ }, "browserify-rsa": { "version": "4.0.1", - "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", + "resolved": "http://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=", "dev": true, "requires": { @@ -2388,7 +2388,7 @@ }, "buffer": { "version": "4.9.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "resolved": "http://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", "dev": true, "requires": { @@ -2551,7 +2551,7 @@ }, "camelcase-keys": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", + "resolved": "http://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=", "dev": true, "requires": { @@ -2600,7 +2600,7 @@ }, "chalk": { "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", "requires": { "ansi-styles": "^2.2.1", @@ -3133,7 +3133,7 @@ }, "create-hash": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", "dev": true, "requires": { @@ -3146,7 +3146,7 @@ }, "create-hmac": { "version": "1.1.7", - "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", + "resolved": "http://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", "dev": true, "requires": { @@ -3293,7 +3293,7 @@ }, "css-select": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", "dev": true, "requires": { @@ -3661,7 +3661,7 @@ }, "diffie-hellman": { "version": "5.0.3", - "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", + "resolved": "http://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==", "dev": true, "requires": { @@ -3725,7 +3725,7 @@ "dependencies": { "domelementtype": { "version": "1.1.3", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", + "resolved": "http://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=", "dev": true }, @@ -3930,7 +3930,7 @@ }, "entities": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/entities/-/entities-1.0.0.tgz", "integrity": "sha1-sph6o4ITR/zeZCsk/fyeT7cSvyY=", "dev": true }, @@ -4353,7 +4353,7 @@ }, "eventemitter2": { "version": "0.4.14", - "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", + "resolved": "http://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz", "integrity": "sha1-j2G3XN4BKy6esoTUVFWDtWQ7Yas=", "dev": true }, @@ -4365,7 +4365,7 @@ }, "events": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "resolved": "http://registry.npmjs.org/events/-/events-1.1.1.tgz", "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", "dev": true }, @@ -4794,7 +4794,7 @@ }, "finalhandler": { "version": "1.1.1", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", "dev": true, "requires": { @@ -5030,7 +5030,7 @@ }, "fs-extra": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz", "integrity": "sha1-zTzl9+fLYUWIP8rjGR6Yd/hYeVA=", "dev": true, "requires": { @@ -5077,7 +5077,8 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "aproba": { "version": "1.2.0", @@ -5098,12 +5099,14 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5118,17 +5121,20 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", @@ -5245,7 +5251,8 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "ini": { "version": "1.3.5", @@ -5257,6 +5264,7 @@ "version": "1.0.0", "bundled": true, "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -5271,6 +5279,7 @@ "version": "3.0.4", "bundled": true, "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5278,12 +5287,14 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "minipass": { "version": "2.2.4", "bundled": true, "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.1.1", "yallist": "^3.0.0" @@ -5302,6 +5313,7 @@ "version": "0.5.1", "bundled": true, "dev": true, + "optional": true, "requires": { "minimist": "0.0.8" } @@ -5382,7 +5394,8 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "object-assign": { "version": "4.1.1", @@ -5394,6 +5407,7 @@ "version": "1.4.0", "bundled": true, "dev": true, + "optional": true, "requires": { "wrappy": "1" } @@ -5479,7 +5493,8 @@ "safe-buffer": { "version": "5.1.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "safer-buffer": { "version": "2.1.2", @@ -5515,6 +5530,7 @@ "version": "1.0.2", "bundled": true, "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -5534,6 +5550,7 @@ "version": "3.0.1", "bundled": true, "dev": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -5577,12 +5594,14 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "yallist": { "version": "3.0.2", "bundled": true, - "dev": true + "dev": true, + "optional": true } } }, @@ -5694,7 +5713,7 @@ }, "get-stream": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", "dev": true }, @@ -5836,7 +5855,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -5913,7 +5932,7 @@ }, "grunt-cli": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/grunt-cli/-/grunt-cli-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/grunt-cli/-/grunt-cli-1.2.0.tgz", "integrity": "sha1-VisRnrsGndtGSs4oRVAb6Xs1tqg=", "dev": true, "requires": { @@ -5961,7 +5980,7 @@ "dependencies": { "shelljs": { "version": "0.5.3", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz", + "resolved": "http://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz", "integrity": "sha1-xUmCuZbHbvDB5rWfvcWCX1txMRM=", "dev": true } @@ -5981,7 +6000,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -6026,7 +6045,7 @@ }, "grunt-contrib-jshint": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/grunt-contrib-jshint/-/grunt-contrib-jshint-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/grunt-contrib-jshint/-/grunt-contrib-jshint-1.1.0.tgz", "integrity": "sha1-Np2QmyWTxA6L55lAshNAhQx5Oaw=", "dev": true, "requires": { @@ -6125,7 +6144,7 @@ "dependencies": { "colors": { "version": "1.1.2", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-1.1.2.tgz", "integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM=", "dev": true } @@ -6189,7 +6208,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -6506,7 +6525,7 @@ }, "htmlparser2": { "version": "3.8.3", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", + "resolved": "http://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", "integrity": "sha1-mWwosZFRaovoZQGn15dX5ccMEGg=", "dev": true, "requires": { @@ -6525,7 +6544,7 @@ }, "http-errors": { "version": "1.6.3", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", "dev": true, "requires": { @@ -6575,7 +6594,7 @@ }, "http-proxy-middleware": { "version": "0.18.0", - "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", + "resolved": "http://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", "integrity": "sha512-Fs25KVMPAIIcgjMZkVHJoKg9VcXcC1C8yb9JUgeDvVXY0S/zgVIhMb+qVswDIgtJe2DfckMSY2d6TuTEutlk6Q==", "dev": true, "requires": { @@ -7011,7 +7030,7 @@ }, "is-builtin-module": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=", "dev": true, "requires": { @@ -7572,7 +7591,7 @@ }, "jsonfile": { "version": "2.4.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", + "resolved": "http://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=", "dev": true, "requires": { @@ -7683,7 +7702,7 @@ }, "kew": { "version": "0.7.0", - "resolved": "https://registry.npmjs.org/kew/-/kew-0.7.0.tgz", + "resolved": "http://registry.npmjs.org/kew/-/kew-0.7.0.tgz", "integrity": "sha1-edk9LTM2PW/dKXCzNdkUGtWR15s=", "dev": true }, @@ -7802,7 +7821,7 @@ }, "load-json-file": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", "dev": true, "requires": { @@ -7815,7 +7834,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -8179,7 +8198,7 @@ }, "media-typer": { "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=", "dev": true }, @@ -8238,7 +8257,7 @@ }, "meow": { "version": "3.7.0", - "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz", + "resolved": "http://registry.npmjs.org/meow/-/meow-3.7.0.tgz", "integrity": "sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=", "dev": true, "requires": { @@ -8435,7 +8454,7 @@ }, "mkdirp": { "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", "requires": { "minimist": "0.0.8" @@ -8476,7 +8495,7 @@ "dependencies": { "commander": { "version": "2.15.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz", + "resolved": "http://registry.npmjs.org/commander/-/commander-2.15.1.tgz", "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==", "dev": true, "optional": true @@ -8645,7 +8664,7 @@ }, "ncp": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/ncp/-/ncp-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/ncp/-/ncp-1.0.1.tgz", "integrity": "sha1-0VNn5cuHQyuhF9K/gP30Wuz7QkY=", "dev": true }, @@ -8744,7 +8763,7 @@ "dependencies": { "semver": { "version": "5.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz", + "resolved": "http://registry.npmjs.org/semver/-/semver-5.3.0.tgz", "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=", "dev": true } @@ -8927,7 +8946,7 @@ "dependencies": { "colors": { "version": "0.5.1", - "resolved": "https://registry.npmjs.org/colors/-/colors-0.5.1.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-0.5.1.tgz", "integrity": "sha1-fQAj6usVTo7p/Oddy5I9DtFmd3Q=" }, "underscore": { @@ -9221,13 +9240,13 @@ }, "os-homedir": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "resolved": "http://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", "dev": true }, "os-locale": { "version": "1.4.0", - "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", + "resolved": "http://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=", "dev": true, "requires": { @@ -9236,7 +9255,7 @@ }, "os-tmpdir": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "resolved": "http://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "dev": true }, @@ -9451,7 +9470,7 @@ }, "parse-asn1": { "version": "5.1.1", - "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", + "resolved": "http://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz", "integrity": "sha512-KPx7flKXg775zZpnp9SxJlz00gTd4BmJ2yJufSc44gMCRrRQ7NSzAcSJQfifuOLgW6bEi+ftrALtsgALeB2Adw==", "dev": true, "requires": { @@ -9537,7 +9556,7 @@ }, "path-is-absolute": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "dev": true }, @@ -9578,7 +9597,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -9761,7 +9780,7 @@ "dependencies": { "async": { "version": "1.5.2", - "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true } @@ -10132,7 +10151,7 @@ }, "progress": { "version": "1.1.8", - "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz", + "resolved": "http://registry.npmjs.org/progress/-/progress-1.1.8.tgz", "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=" }, "promise-inflight": { @@ -10157,13 +10176,13 @@ "dependencies": { "async": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz", "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=", "dev": true }, "winston": { "version": "2.1.1", - "resolved": "https://registry.npmjs.org/winston/-/winston-2.1.1.tgz", + "resolved": "http://registry.npmjs.org/winston/-/winston-2.1.1.tgz", "integrity": "sha1-PJNJ0ZYgf9G9/51LxD73JRDjoS4=", "dev": true, "requires": { @@ -10178,7 +10197,7 @@ "dependencies": { "colors": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true }, @@ -10401,7 +10420,7 @@ "dependencies": { "pify": { "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz", "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", "dev": true } @@ -10590,7 +10609,7 @@ "dependencies": { "jsesc": { "version": "0.5.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", + "resolved": "http://registry.npmjs.org/jsesc/-/jsesc-0.5.0.tgz", "integrity": "sha1-597mbjXW/Bb3EP6R1c9p9w8IkR0=", "dev": true } @@ -10641,7 +10660,7 @@ }, "htmlparser2": { "version": "3.3.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.3.0.tgz", + "resolved": "http://registry.npmjs.org/htmlparser2/-/htmlparser2-3.3.0.tgz", "integrity": "sha1-zHDQWln2VC5D8OaFyYLhTJJKnv4=", "dev": true, "requires": { @@ -10653,7 +10672,7 @@ }, "readable-stream": { "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", "dev": true, "requires": { @@ -10763,7 +10782,7 @@ }, "require-uncached": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", "integrity": "sha1-Tg1W1slmL9MeQwEcS5WqSZVUIdM=", "dev": true, "requires": { @@ -10930,7 +10949,7 @@ }, "safe-regex": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=", "dev": true, "requires": { @@ -11250,7 +11269,7 @@ }, "sha.js": { "version": "2.4.11", - "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", + "resolved": "http://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", "dev": true, "requires": { @@ -11294,7 +11313,7 @@ }, "shelljs": { "version": "0.3.0", - "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", + "resolved": "http://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz", "integrity": "sha1-NZbmMHp4FUT1kfN9phg2DzHbV7E=", "dev": true }, @@ -12015,7 +12034,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "requires": { "ansi-regex": "^2.0.0" @@ -12032,7 +12051,7 @@ }, "strip-eof": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", "dev": true }, @@ -12125,7 +12144,7 @@ }, "tar": { "version": "2.2.1", - "resolved": "https://registry.npmjs.org/tar/-/tar-2.2.1.tgz", + "resolved": "http://registry.npmjs.org/tar/-/tar-2.2.1.tgz", "integrity": "sha1-jk0qJWwOIYXGsYrWlK7JaLg8sdE=", "dev": true, "requires": { @@ -12283,7 +12302,7 @@ }, "through": { "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", "dev": true }, @@ -12943,7 +12962,7 @@ "dependencies": { "async": { "version": "0.9.2", - "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-0.9.2.tgz", "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=", "dev": true }, @@ -12969,7 +12988,7 @@ }, "valid-data-url": { "version": "0.1.6", - "resolved": "https://registry.npmjs.org/valid-data-url/-/valid-data-url-0.1.6.tgz", + "resolved": "http://registry.npmjs.org/valid-data-url/-/valid-data-url-0.1.6.tgz", "integrity": "sha512-FXg2qXMzfAhZc0y2HzELNfUeiOjPr+52hU1DNBWiJJ2luXD+dD1R9NA48Ug5aj0ibbxroeGDc/RJv6ThiGgkDw==", "dev": true }, @@ -12985,7 +13004,7 @@ }, "validator": { "version": "9.4.1", - "resolved": "https://registry.npmjs.org/validator/-/validator-9.4.1.tgz", + "resolved": "http://registry.npmjs.org/validator/-/validator-9.4.1.tgz", "integrity": "sha512-YV5KjzvRmSyJ1ee/Dm5UED0G+1L4GZnLN3w6/T+zZm8scVua4sOhYKWTUrKa0H/tMiJyO9QLHMPN+9mB/aMunA==", "dev": true }, @@ -13517,7 +13536,7 @@ }, "webpack-node-externals": { "version": "1.7.2", - "resolved": "https://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.7.2.tgz", + "resolved": "http://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.7.2.tgz", "integrity": "sha512-ajerHZ+BJKeCLviLUUmnyd5B4RavLF76uv3cs6KNuO8W+HuQaEs0y0L7o40NQxdPy5w0pcv8Ew7yPUAQG0UdCg==", "dev": true }, @@ -13671,14 +13690,14 @@ "dependencies": { "async": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz", "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=", "dev": true, "optional": true }, "colors": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz", "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=", "dev": true, "optional": true @@ -13711,7 +13730,7 @@ }, "wrap-ansi": { "version": "2.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", + "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=", "dev": true, "requires": { diff --git a/src/core/operations/YARARules.mjs b/src/core/operations/YARARules.mjs index c453d6b8..91a27963 100644 --- a/src/core/operations/YARARules.mjs +++ b/src/core/operations/YARARules.mjs @@ -35,25 +35,21 @@ class YARARules extends Operation { { name: "Show strings", type: "boolean", - hint: "Show each match's data", value: false }, { name: "Show string lengths", type: "boolean", - hint: "Show the length of each match's data", value: false }, { name: "Show metadata", type: "boolean", - hint: "Show the metadata of each rule", value: false }, { name: "Show counts", type: "boolean", - hint: "Show the number of matches per rule", value: true } ]; @@ -66,7 +62,7 @@ class YARARules extends Operation { */ run(input, args) { if (ENVIRONMENT_IS_WORKER()) - self.sendStatusMessage("Instantiating YARA."); + self.sendStatusMessage("Instantiating YARA..."); const [rules, showStrings, showLengths, showMeta, showCounts] = args; return new Promise((resolve, reject) => { Yara().then(yara => { diff --git a/src/web/RecipeWaiter.mjs b/src/web/RecipeWaiter.mjs index a7a9e2e3..4eff5d9c 100755 --- a/src/web/RecipeWaiter.mjs +++ b/src/web/RecipeWaiter.mjs @@ -376,6 +376,7 @@ class RecipeWaiter { } } + /** * Adds the specified operation to the recipe. * @@ -453,6 +454,7 @@ class RecipeWaiter { window.dispatchEvent(this.manager.statechange); } + /** * Handler for text argument dragover events. * Gives the user a visual cue to show that items can be dropped here. @@ -469,6 +471,7 @@ class RecipeWaiter { e.target.closest("textarea.arg").classList.add("dropping-file"); } + /** * Handler for text argument dragleave events. * Removes the visual cue. @@ -481,6 +484,7 @@ class RecipeWaiter { e.target.classList.remove("dropping-file"); } + /** * Handler for text argument drop events. * Loads the dragged data into the argument textarea. @@ -510,7 +514,10 @@ class RecipeWaiter { const self = this; reader.onload = function (e) { targ.value = e.target.result; - self.ingChange(); + // Trigger floating label move + const changeEvent = new Event('change'); + targ.dispatchEvent(changeEvent); + window.dispatchEvent(self.manager.statechange); }; reader.readAsText(file); } @@ -542,6 +549,7 @@ class RecipeWaiter { op.insertAdjacentHTML("beforeend", registerListEl); } + /** * Adjusts the number of ingredient columns as the width of the recipe changes. */ diff --git a/src/web/stylesheets/layout/_io.css b/src/web/stylesheets/layout/_io.css index 5b0433f6..83d37aaf 100755 --- a/src/web/stylesheets/layout/_io.css +++ b/src/web/stylesheets/layout/_io.css @@ -123,6 +123,7 @@ .dropping-file { border: 5px dashed var(--drop-file-border-colour) !important; + margin: -5px; } #stale-indicator { From 715f7bbbc2900e20c9130241d098f3720890c4a8 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 18 Jan 2019 15:37:25 +0000 Subject: [PATCH 50/52] Lint --- src/web/RecipeWaiter.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web/RecipeWaiter.mjs b/src/web/RecipeWaiter.mjs index 4eff5d9c..2b0b0a05 100755 --- a/src/web/RecipeWaiter.mjs +++ b/src/web/RecipeWaiter.mjs @@ -515,7 +515,7 @@ class RecipeWaiter { reader.onload = function (e) { targ.value = e.target.result; // Trigger floating label move - const changeEvent = new Event('change'); + const changeEvent = new Event("change"); targ.dispatchEvent(changeEvent); window.dispatchEvent(self.manager.statechange); }; From eca2c142f3b053b0d8ab00c3dfe674799d8ad583 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 18 Jan 2019 15:37:42 +0000 Subject: [PATCH 51/52] 8.23.0 --- package-lock.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 469dfa1e..57ff15fc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.22.1", + "version": "8.23.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index beccd97c..e6f62093 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cyberchef", - "version": "8.22.1", + "version": "8.23.0", "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.", "author": "n1474335 ", "homepage": "https://gchq.github.io/CyberChef", From 69837837b077c2d4212235f2c65e4f19bd135f52 Mon Sep 17 00:00:00 2001 From: n1474335 Date: Fri, 18 Jan 2019 17:14:25 +0000 Subject: [PATCH 52/52] Tidied up co-ordinate operation --- CHANGELOG.md | 7 +- src/core/lib/ConvertCoordinates.mjs | 354 +++++++++--------- .../operations/ConvertCoordinateFormat.mjs | 11 +- 3 files changed, 195 insertions(+), 177 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8f11e2e..3ad915aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,11 @@ All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master). +### [8.24.0] - 2019-01-18 +- 'Convert co-ordinate format' operation added [@j433866] | [#476] + ### [8.23.0] - 2019-01-18 -- 'YARA Rules' operatio added [@artemisbot] | [#468] +- 'YARA Rules' operation added [@artemisbot] | [#468] ### [8.22.0] - 2019-01-10 - 'Subsection' operation added [@j433866] | [#467] @@ -100,6 +103,7 @@ All major and minor version changes will be documented in this file. Details of +[8.24.0]: https://github.com/gchq/CyberChef/releases/tag/v8.24.0 [8.23.0]: https://github.com/gchq/CyberChef/releases/tag/v8.23.0 [8.22.0]: https://github.com/gchq/CyberChef/releases/tag/v8.22.0 [8.21.0]: https://github.com/gchq/CyberChef/releases/tag/v8.21.0 @@ -181,3 +185,4 @@ All major and minor version changes will be documented in this file. Details of [#461]: https://github.com/gchq/CyberChef/pull/461 [#467]: https://github.com/gchq/CyberChef/pull/467 [#468]: https://github.com/gchq/CyberChef/pull/468 +[#476]: https://github.com/gchq/CyberChef/pull/476 diff --git a/src/core/lib/ConvertCoordinates.mjs b/src/core/lib/ConvertCoordinates.mjs index d4641fe6..ad592b74 100644 --- a/src/core/lib/ConvertCoordinates.mjs +++ b/src/core/lib/ConvertCoordinates.mjs @@ -8,6 +8,7 @@ import geohash from "ngeohash"; import geodesy from "geodesy"; +import OperationError from "../errors/OperationError"; /** * Co-ordinate formats @@ -34,6 +35,7 @@ const NO_CHANGE = [ /** * Convert a given latitude and longitude into a different format. + * * @param {string} input - Input string to be converted * @param {string} inFormat - Format of the input coordinates * @param {string} inDelim - The delimiter splitting the lat/long of the input @@ -49,29 +51,40 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli latlon, convLat, convLon, - conv; + conv, + hash, + utm, + mgrs, + osng, + splitLat, + splitLong, + lat, + lon; // Can't have a precision less than 0! if (precision < 0) { precision = 0; } + if (inDelim === "Auto") { // Try to detect a delimiter in the input. inDelim = findDelim(input); if (inDelim === null) { - throw "Unable to detect the input delimiter automatically."; + throw new OperationError("Unable to detect the input delimiter automatically."); } } else { // Convert the delimiter argument value to the actual character inDelim = realDelim(inDelim); } + if (inFormat === "Auto") { // Try to detect the format of the input data inFormat = findFormat(input, inDelim); if (inFormat === null) { - throw "Unable to detect the input format automatically."; + throw new OperationError("Unable to detect the input format automatically."); } } + // Convert the output delimiter argument to the real character outDelim = realDelim(outDelim); @@ -91,83 +104,91 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli } // Conversions from the input format into a geodesy latlon object - if (inFormat === "Geohash") { - const hash = geohash.decode(input.replace(/[^A-Za-z0-9]/g, "")); - latlon = new geodesy.LatLonEllipsoidal(hash.latitude, hash.longitude); - } else if (inFormat === "Military Grid Reference System") { - const utm = geodesy.Mgrs.parse(input.replace(/[^A-Za-z0-9]/g, "")).toUtm(); - latlon = utm.toLatLonE(); - } else if (inFormat === "Ordnance Survey National Grid") { - const osng = geodesy.OsGridRef.parse(input.replace(/[^A-Za-z0-9]/g, "")); - latlon = geodesy.OsGridRef.osGridToLatLon(osng); - } else if (inFormat === "Universal Transverse Mercator") { - // Geodesy needs a space between the first 2 digits and the next letter - if (/^[\d]{2}[A-Za-z]/.test(input)) { - input = input.slice(0, 2) + " " + input.slice(2); - } - const utm = geodesy.Utm.parse(input); - latlon = utm.toLatLonE(); - } else if (inFormat === "Degrees Minutes Seconds") { - if (isPair) { - // Split up the lat/long into degrees / minutes / seconds values - const splitLat = splitInput(split[0]), + switch (inFormat) { + case "Geohash": + hash = geohash.decode(input.replace(/[^A-Za-z0-9]/g, "")); + latlon = new geodesy.LatLonEllipsoidal(hash.latitude, hash.longitude); + break; + case "Military Grid Reference System": + utm = geodesy.Mgrs.parse(input.replace(/[^A-Za-z0-9]/g, "")).toUtm(); + latlon = utm.toLatLonE(); + break; + case "Ordnance Survey National Grid": + osng = geodesy.OsGridRef.parse(input.replace(/[^A-Za-z0-9]/g, "")); + latlon = geodesy.OsGridRef.osGridToLatLon(osng); + break; + case "Universal Transverse Mercator": + // Geodesy needs a space between the first 2 digits and the next letter + if (/^[\d]{2}[A-Za-z]/.test(input)) { + input = input.slice(0, 2) + " " + input.slice(2); + } + utm = geodesy.Utm.parse(input); + latlon = utm.toLatLonE(); + break; + case "Degrees Minutes Seconds": + if (isPair) { + // Split up the lat/long into degrees / minutes / seconds values + splitLat = splitInput(split[0]); splitLong = splitInput(split[1]); - if (splitLat.length >= 3 && splitLong.length >= 3) { - const lat = convDMSToDD(splitLat[0], splitLat[1], splitLat[2], 10); - const long = convDMSToDD(splitLong[0], splitLong[1], splitLong[2], 10); - latlon = new geodesy.LatLonEllipsoidal(lat.degrees, long.degrees); + if (splitLat.length >= 3 && splitLong.length >= 3) { + lat = convDMSToDD(splitLat[0], splitLat[1], splitLat[2], 10); + lon = convDMSToDD(splitLong[0], splitLong[1], splitLong[2], 10); + latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lon.degrees); + } else { + throw new OperationError("Invalid co-ordinate format for Degrees Minutes Seconds"); + } } else { - throw "Invalid co-ordinate format for Degrees Minutes Seconds"; + // Not a pair, so only try to convert one set of co-ordinates + splitLat = splitInput(split[0]); + if (splitLat.length >= 3) { + lat = convDMSToDD(splitLat[0], splitLat[1], splitLat[2]); + latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lat.degrees); + } else { + throw new OperationError("Invalid co-ordinate format for Degrees Minutes Seconds"); + } } - } else { - // Not a pair, so only try to convert one set of co-ordinates - const splitLat = splitInput(split[0]); - if (splitLat.length >= 3) { - const lat = convDMSToDD(splitLat[0], splitLat[1], splitLat[2]); + break; + case "Degrees Decimal Minutes": + if (isPair) { + splitLat = splitInput(split[0]); + splitLong = splitInput(split[1]); + if (splitLat.length !== 2 || splitLong.length !== 2) { + throw new OperationError("Invalid co-ordinate format for Degrees Decimal Minutes."); + } + // Convert to decimal degrees, and then convert to a geodesy object + lat = convDDMToDD(splitLat[0], splitLat[1], 10); + lon = convDDMToDD(splitLong[0], splitLong[1], 10); + latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lon.degrees); + } else { + // Not a pair, so only try to convert one set of co-ordinates + splitLat = splitInput(input); + if (splitLat.length !== 2) { + throw new OperationError("Invalid co-ordinate format for Degrees Decimal Minutes."); + } + lat = convDDMToDD(splitLat[0], splitLat[1], 10); latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lat.degrees); + } + break; + case "Decimal Degrees": + if (isPair) { + splitLat = splitInput(split[0]); + splitLong = splitInput(split[1]); + if (splitLat.length !== 1 || splitLong.length !== 1) { + throw new OperationError("Invalid co-ordinate format for Decimal Degrees."); + } + latlon = new geodesy.LatLonEllipsoidal(splitLat[0], splitLong[0]); } else { - throw "Invalid co-ordinate format for Degrees Minutes Seconds"; + // Not a pair, so only try to convert one set of co-ordinates + splitLat = splitInput(split[0]); + if (splitLat.length !== 1) { + throw new OperationError("Invalid co-ordinate format for Decimal Degrees."); + } + latlon = new geodesy.LatLonEllipsoidal(splitLat[0], splitLat[0]); } - } - } else if (inFormat === "Degrees Decimal Minutes") { - if (isPair) { - const splitLat = splitInput(split[0]); - const splitLong = splitInput(split[1]); - if (splitLat.length !== 2 || splitLong.length !== 2) { - throw "Invalid co-ordinate format for Degrees Decimal Minutes."; - } - // Convert to decimal degrees, and then convert to a geodesy object - const lat = convDDMToDD(splitLat[0], splitLat[1], 10); - const long = convDDMToDD(splitLong[0], splitLong[1], 10); - latlon = new geodesy.LatLonEllipsoidal(lat.degrees, long.degrees); - } else { - // Not a pair, so only try to convert one set of co-ordinates - const splitLat = splitInput(input); - if (splitLat.length !== 2) { - throw "Invalid co-ordinate format for Degrees Decimal Minutes."; - } - const lat = convDDMToDD(splitLat[0], splitLat[1], 10); - latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lat.degrees); - } - } else if (inFormat === "Decimal Degrees") { - if (isPair) { - const splitLat = splitInput(split[0]); - const splitLong = splitInput(split[1]); - if (splitLat.length !== 1 || splitLong.length !== 1) { - throw "Invalid co-ordinate format for Decimal Degrees."; - } - latlon = new geodesy.LatLonEllipsoidal(splitLat[0], splitLong[0]); - } else { - // Not a pair, so only try to convert one set of co-ordinates - const splitLat = splitInput(split[0]); - if (splitLat.length !== 1) { - throw "Invalid co-ordinate format for Decimal Degrees."; - } - latlon = new geodesy.LatLonEllipsoidal(splitLat[0], splitLat[0]); - } - } else { - throw `Unknown input format '${inFormat}'`; + break; + default: + throw new OperationError(`Unknown input format '${inFormat}'`); } // Everything is now a geodesy latlon object @@ -179,68 +200,78 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli if (dirs && dirs.length >= 1) { // Make positive lat/lon values with S/W directions into negative values if (dirs[0] === "S" || dirs[0] === "W" && latlon.lat > 0) { - latlon.lat = 0 - latlon.lat; + latlon.lat = -latlon.lat; } if (dirs.length >= 2) { if (dirs[1] === "S" || dirs[1] === "W" && latlon.lon > 0) { - latlon.lon = 0 - latlon.lon; + latlon.lon = -latlon.lon; } } } } + // Try to find the compass directions of the lat and long const [latDir, longDir] = findDirs(latlon.lat + "," + latlon.lon, ","); + // Output conversions for each output format - if (outFormat === "Decimal Degrees") { - // We could use the built in latlon.toString(), - // but this makes adjusting the output harder - const lat = convDDToDD(latlon.lat, precision); - const lon = convDDToDD(latlon.lon, precision); - convLat = lat.string; - convLon = lon.string; - } else if (outFormat === "Degrees Decimal Minutes") { - const lat = convDDToDDM(latlon.lat, precision); - const lon = convDDToDDM(latlon.lon, precision); - convLat = lat.string; - convLon = lon.string; - } else if (outFormat === "Degrees Minutes Seconds") { - const lat = convDDToDMS(latlon.lat, precision); - const lon = convDDToDMS(latlon.lon, precision); - convLat = lat.string; - convLon = lon.string; - } else if (outFormat === "Geohash") { - convLat = geohash.encode(latlon.lat, latlon.lon, precision); - } else if (outFormat === "Military Grid Reference System") { - const utm = latlon.toUtm(); - const mgrs = utm.toMgrs(); - // MGRS wants a precision that's an even number between 2 and 10 - if (precision % 2 !== 0) { - precision = precision + 1; - } - if (precision > 10) { - precision = 10; - } - convLat = mgrs.toString(precision); - } else if (outFormat === "Ordnance Survey National Grid") { - const osng = geodesy.OsGridRef.latLonToOsGrid(latlon); - if (osng.toString() === "") { - throw "Could not convert co-ordinates to OS National Grid. Are the co-ordinates in range?"; - } - // OSNG wants a precision that's an even number between 2 and 10 - if (precision % 2 !== 0) { - precision = precision + 1; - } - if (precision > 10) { - precision = 10; - } - convLat = osng.toString(precision); - } else if (outFormat === "Universal Transverse Mercator") { - const utm = latlon.toUtm(); - convLat = utm.toString(precision); + switch (outFormat) { + case "Decimal Degrees": + // We could use the built in latlon.toString(), + // but this makes adjusting the output harder + lat = convDDToDD(latlon.lat, precision); + lon = convDDToDD(latlon.lon, precision); + convLat = lat.string; + convLon = lon.string; + break; + case "Degrees Decimal Minutes": + lat = convDDToDDM(latlon.lat, precision); + lon = convDDToDDM(latlon.lon, precision); + convLat = lat.string; + convLon = lon.string; + break; + case "Degrees Minutes Seconds": + lat = convDDToDMS(latlon.lat, precision); + lon = convDDToDMS(latlon.lon, precision); + convLat = lat.string; + convLon = lon.string; + break; + case "Geohash": + convLat = geohash.encode(latlon.lat, latlon.lon, precision); + break; + case "Military Grid Reference System": + utm = latlon.toUtm(); + mgrs = utm.toMgrs(); + // MGRS wants a precision that's an even number between 2 and 10 + if (precision % 2 !== 0) { + precision = precision + 1; + } + if (precision > 10) { + precision = 10; + } + convLat = mgrs.toString(precision); + break; + case "Ordnance Survey National Grid": + osng = geodesy.OsGridRef.latLonToOsGrid(latlon); + if (osng.toString() === "") { + throw new OperationError("Could not convert co-ordinates to OS National Grid. Are the co-ordinates in range?"); + } + // OSNG wants a precision that's an even number between 2 and 10 + if (precision % 2 !== 0) { + precision = precision + 1; + } + if (precision > 10) { + precision = 10; + } + convLat = osng.toString(precision); + break; + case "Universal Transverse Mercator": + utm = latlon.toUtm(); + convLat = utm.toString(precision); + break; } if (convLat === undefined) { - throw "Error converting co-ordinates."; + throw new OperationError("Error converting co-ordinates."); } if (outFormat.includes("Degrees")) { @@ -283,6 +314,7 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli /** * Split up the input using a space or degrees signs, and sanitise the result + * * @param {string} input - The input data to be split * @returns {number[]} An array of the different items in the string, stored as floats */ @@ -302,6 +334,7 @@ function splitInput (input){ /** * Convert Degrees Minutes Seconds to Decimal Degrees + * * @param {number} degrees - The degrees of the input co-ordinates * @param {number} minutes - The minutes of the input co-ordinates * @param {number} seconds - The seconds of the input co-ordinates @@ -324,6 +357,7 @@ function convDMSToDD (degrees, minutes, seconds, precision){ /** * Convert Decimal Degrees Minutes to Decimal Degrees + * * @param {number} degrees - The input degrees to be converted * @param {number} minutes - The input minutes to be converted * @param {number} precision - The precision which the result should be rounded to @@ -345,6 +379,7 @@ function convDDMToDD (degrees, minutes, precision) { /** * Convert Decimal Degrees to Decimal Degrees + * * Doesn't affect the input, just puts it into an object * @param {number} degrees - The input degrees to be converted * @param {number} precision - The precision which the result should be rounded to @@ -359,6 +394,7 @@ function convDDToDD (degrees, precision) { /** * Convert Decimal Degrees to Degrees Minutes Seconds + * * @param {number} decDegrees - The input data to be converted * @param {number} precision - The precision which the result should be rounded to * @returns {{string: string, degrees: number, minutes: number, seconds: number}} An object containing the raw converted value as separate numbers (.degrees, .minutes, .seconds), and a formatted string version (obj.string) @@ -383,6 +419,7 @@ function convDDToDMS (decDegrees, precision) { /** * Convert Decimal Degrees to Degrees Decimal Minutes + * * @param {number} decDegrees - The input degrees to be converted * @param {number} precision - The precision the input data should be rounded to * @returns {{string: string, degrees: number, minutes: number}} An object containing the raw converted value as separate numbers (.degrees, .minutes), and a formatted string version (obj.string) @@ -407,6 +444,7 @@ function convDDToDDM (decDegrees, precision) { /** * Finds and returns the compass directions in an input string + * * @param {string} input - The input co-ordinates containing the direction * @param {string} delim - The delimiter separating latitide and longitude * @returns {string[]} String array containing the latitude and longitude directions @@ -421,13 +459,10 @@ export function findDirs(input, delim) { // If there's actually compass directions // in the input, use these to work out the direction if (dirs.length <= 2 && dirs.length >= 1) { - if (dirs.length === 2) { - return [dirs[0], dirs[1]]; - } else { - return [dirs[0], ""]; - } + return dirs.length === 2 ? [dirs[0], dirs[1]] : [dirs[0], ""]; } } + // Nothing was returned, so guess the directions let lat = upperInput, long, @@ -440,43 +475,29 @@ export function findDirs(input, delim) { if (split[0] !== "") { lat = split[0]; } - if (split.length >= 2) { - if (split[1] !== "") { - long = split[1]; - } + if (split.length >= 2 && split[1] !== "") { + long = split[1]; } } } } else { const split = upperInput.split(dirExp); if (split.length > 1) { - if (split[0] === "") { - lat = split[1]; - } else { - lat = split[0]; - } - if (split.length > 2) { - if (split[2] !== "") { - long = split[2]; - } + lat = split[0] === "" ? split[1] : split[0]; + if (split.length > 2 && split[2] !== "") { + long = split[2]; } } } + if (lat) { lat = parseFloat(lat); - if (lat < 0) { - latDir = "S"; - } else { - latDir = "N"; - } + latDir = lat < 0 ? "S" : "N"; } + if (long) { long = parseFloat(long); - if (long < 0) { - longDir = "W"; - } else { - longDir = "E"; - } + longDir = long < 0 ? "W" : "E"; } return [latDir, longDir]; @@ -484,6 +505,7 @@ export function findDirs(input, delim) { /** * Detects the co-ordinate format of the input data + * * @param {string} input - The input data whose format we need to detect * @param {string} delim - The delimiter separating the data in input * @returns {string} The input format @@ -495,25 +517,19 @@ export function findFormat (input, delim) { geohashPattern = new RegExp(/^[0123456789BCDEFGHJKMNPQRSTUVWXYZ]+$/), utmPattern = new RegExp(/^[0-9]{2}\s?[C-HJ-NP-X]\s[0-9.]+\s?[0-9.]+$/), degPattern = new RegExp(/[°'"]/g); + input = input.trim(); + if (delim !== null && delim.includes("Direction")) { const split = input.split(/[NnEeSsWw]/); if (split.length > 1) { - if (split[0] === "") { - testData = split[1]; - } else { - testData = split[0]; - } + testData = split[0] === "" ? split[1] : split[0]; } } else if (delim !== null && delim !== "") { if (input.includes(delim)) { const split = input.split(delim); if (split.length > 1) { - if (split[0] === "") { - testData = split[1]; - } else { - testData = split[0]; - } + testData = split[0] === "" ? split[1] : split[0]; } } else { testData = input; @@ -523,20 +539,17 @@ export function findFormat (input, delim) { // Test non-degrees formats if (!degPattern.test(input)) { const filteredInput = input.toUpperCase().replace(delim, ""); - const isMgrs = mgrsPattern.test(filteredInput); - const isOsng = osngPattern.test(filteredInput); - const isGeohash = geohashPattern.test(filteredInput); - const isUtm = utmPattern.test(filteredInput); - if (isUtm) { + + if (utmPattern.test(filteredInput)) { return "Universal Transverse Mercator"; } - if (isMgrs) { + if (mgrsPattern.test(filteredInput)) { return "Military Grid Reference System"; } - if (isOsng) { + if (osngPattern.test(filteredInput)) { return "Ordnance Survey National Grid"; } - if (isGeohash) { + if (geohashPattern.test(filteredInput)) { return "Geohash"; } } @@ -558,6 +571,7 @@ export function findFormat (input, delim) { /** * Automatically find the delimeter type from the given input + * * @param {string} input * @returns {string} Delimiter type */ @@ -594,6 +608,7 @@ export function findDelim (input) { /** * Gets the real string for a delimiter name. + * * @param {string} delim The delimiter to be matched * @returns {string} */ @@ -610,7 +625,9 @@ export function realDelim (delim) { /** * Returns true if a zero is negative + * * @param {number} zero + * @returns {boolean} */ function isNegativeZero(zero) { return zero === 0 && (1/zero < 0); @@ -618,6 +635,7 @@ function isNegativeZero(zero) { /** * Rounds a number to a specified number of decimal places + * * @param {number} input - The number to be rounded * @param {precision} precision - The number of decimal places the number should be rounded to * @returns {number} diff --git a/src/core/operations/ConvertCoordinateFormat.mjs b/src/core/operations/ConvertCoordinateFormat.mjs index 09e1620c..87e44bf2 100644 --- a/src/core/operations/ConvertCoordinateFormat.mjs +++ b/src/core/operations/ConvertCoordinateFormat.mjs @@ -5,7 +5,6 @@ */ import Operation from "../Operation"; -import OperationError from "../errors/OperationError"; import {FORMATS, convertCoordinates} from "../lib/ConvertCoordinates"; /** @@ -21,7 +20,7 @@ class ConvertCoordinateFormat extends Operation { this.name = "Convert co-ordinate format"; this.module = "Hashing"; - this.description = "Convert geographical coordinates between different formats.

Supported formats:
  • Degrees Minutes Seconds (DMS)
  • Degrees Decimal Minutes (DDM)
  • Decimal Degrees (DD)
  • Geohash
  • Military Grid Reference System (MGRS)
  • Ordnance Survey National Grid (OSNG)
  • Universal Transverse Mercator (UTM)

The operation can try to detect the input co-ordinate format and delimiter automatically, but this may not always work correctly."; + this.description = "Converts geographical coordinates between different formats.

Supported formats:
  • Degrees Minutes Seconds (DMS)
  • Degrees Decimal Minutes (DDM)
  • Decimal Degrees (DD)
  • Geohash
  • Military Grid Reference System (MGRS)
  • Ordnance Survey National Grid (OSNG)
  • Universal Transverse Mercator (UTM)

The operation can try to detect the input co-ordinate format and delimiter automatically, but this may not always work correctly."; this.infoURL = "https://wikipedia.org/wiki/Geographic_coordinate_conversion"; this.inputType = "string"; this.outputType = "string"; @@ -85,12 +84,8 @@ class ConvertCoordinateFormat extends Operation { run(input, args) { if (input.replace(/[\s+]/g, "") !== "") { const [inFormat, inDelim, outFormat, outDelim, incDirection, precision] = args; - try { - const result = convertCoordinates(input, inFormat, inDelim, outFormat, outDelim, incDirection, precision); - return result; - } catch (error) { - throw new OperationError(error); - } + const result = convertCoordinates(input, inFormat, inDelim, outFormat, outDelim, incDirection, precision); + return result; } else { return input; }