diff --git a/Gruntfile.js b/Gruntfile.js index a025e940..fb30e017 100755 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -229,6 +229,9 @@ module.exports = function (grunt) { stats: { children: false, warningsFilter: /source-map/ + }, + node: { + fs: "empty" } }, webDev: { diff --git a/package-lock.json b/package-lock.json index 200cd304..ba667663 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1342,6 +1342,11 @@ "integrity": "sha1-vos2rvzN6LPKeqLWr8B6NyQsDS0=", "dev": true }, + "cjson": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/cjson/-/cjson-0.2.1.tgz", + "integrity": "sha1-c82KrWXZ4VBfmvF0TTt5wVJ2gqU=" + }, "clap": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/clap/-/clap-1.2.0.tgz", @@ -2067,6 +2072,11 @@ "domelementtype": "1.3.0" } }, + "ebnf-parser": { + "version": "0.1.10", + "resolved": "https://registry.npmjs.org/ebnf-parser/-/ebnf-parser-0.1.10.tgz", + "integrity": "sha1-zR9rpHfFY4xAyX7ZtXLbW6tdgzE=" + }, "ecc-jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", @@ -4086,6 +4096,52 @@ "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=", "dev": true }, + "jison": { + "version": "0.4.13", + "resolved": "https://registry.npmjs.org/jison/-/jison-0.4.13.tgz", + "integrity": "sha1-kEFwfWIkE2f1iDRTK58ZwsNvrHg=", + "requires": { + "cjson": "0.2.1", + "ebnf-parser": "0.1.10", + "escodegen": "0.0.21", + "esprima": "1.0.4", + "jison-lex": "0.2.1", + "JSONSelect": "0.4.0", + "lex-parser": "0.1.4", + "nomnom": "1.5.2" + }, + "dependencies": { + "escodegen": { + "version": "0.0.21", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-0.0.21.tgz", + "integrity": "sha1-U9ZSz6EDA4gnlFilJmxf/HCcY8M=", + "requires": { + "esprima": "1.0.4", + "estraverse": "0.0.4", + "source-map": "0.2.0" + } + }, + "esprima": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-1.0.4.tgz", + "integrity": "sha1-n1V+CPw7TSbs6d00+Pv0drYlha0=" + }, + "estraverse": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-0.0.4.tgz", + "integrity": "sha1-AaCTLf7ldGhKWYr1pnw7+bZCjbI=" + } + } + }, + "jison-lex": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/jison-lex/-/jison-lex-0.2.1.tgz", + "integrity": "sha1-rEuBXozOUTLrErXfz+jXB7iETf4=", + "requires": { + "lex-parser": "0.1.4", + "nomnom": "1.5.2" + } + }, "jquery": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.2.1.tgz", @@ -4274,12 +4330,40 @@ "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM=", "dev": true }, + "jsonpath": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/jsonpath/-/jsonpath-0.2.12.tgz", + "integrity": "sha1-W/nZEftGFsHjNwvs658NskrjTNI=", + "requires": { + "esprima": "1.2.2", + "jison": "0.4.13", + "static-eval": "0.2.3", + "underscore": "1.7.0" + }, + "dependencies": { + "esprima": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-1.2.2.tgz", + "integrity": "sha1-dqD9Zvz+FU/SkmZ9wmQBl1CxZXs=" + }, + "underscore": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz", + "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=" + } + } + }, "jsonpointer": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-4.0.1.tgz", "integrity": "sha1-T9kss04OnbPInIYi7PUfm5eMbLk=", "dev": true }, + "JSONSelect": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/JSONSelect/-/JSONSelect-0.4.0.tgz", + "integrity": "sha1-oI7cxn6z/L6Z7WMIVTRKDPKCu40=" + }, "jsprim": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.0.tgz", @@ -4397,6 +4481,11 @@ "type-check": "0.3.2" } }, + "lex-parser": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/lex-parser/-/lex-parser-0.1.4.tgz", + "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA=" + }, "load-json-file": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", @@ -4774,6 +4863,27 @@ "vm-browserify": "0.0.4" } }, + "nomnom": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/nomnom/-/nomnom-1.5.2.tgz", + "integrity": "sha1-9DRUSKhTz71cDSYyDyR3qwUm/i8=", + "requires": { + "colors": "0.5.1", + "underscore": "1.1.7" + }, + "dependencies": { + "colors": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/colors/-/colors-0.5.1.tgz", + "integrity": "sha1-fQAj6usVTo7p/Oddy5I9DtFmd3Q=" + }, + "underscore": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.1.7.tgz", + "integrity": "sha1-QLq4S60Z0jAJbo1u9ii/8FXYPbA=" + } + } + }, "nopt": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz", @@ -6811,6 +6921,36 @@ } } }, + "static-eval": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/static-eval/-/static-eval-0.2.3.tgz", + "integrity": "sha1-Aj8XrJ/uQm6niMEuo5IG3Bdfiyo=", + "requires": { + "escodegen": "0.0.28" + }, + "dependencies": { + "escodegen": { + "version": "0.0.28", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-0.0.28.tgz", + "integrity": "sha1-Dk/xcV8yh3XWyrUaxEpAbNer/9M=", + "requires": { + "esprima": "1.0.4", + "estraverse": "1.3.2", + "source-map": "0.2.0" + } + }, + "esprima": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-1.0.4.tgz", + "integrity": "sha1-n1V+CPw7TSbs6d00+Pv0drYlha0=" + }, + "estraverse": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-1.3.2.tgz", + "integrity": "sha1-N8K4k+8T1yPydth41g2FNRUqbEI=" + } + } + }, "stream-browserify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.1.tgz", diff --git a/package.json b/package.json index 2afef7b2..fee8a17b 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,7 @@ "google-code-prettify": "^1.0.5", "jquery": "^3.1.1", "jsbn": "^1.1.0", + "jsonpath": "^0.2.12", "jsrsasign": "8.0.3", "lodash": "^4.17.4", "moment": "^2.17.1", diff --git a/src/core/config/Categories.js b/src/core/config/Categories.js index ce46d221..b3e40b33 100755 --- a/src/core/config/Categories.js +++ b/src/core/config/Categories.js @@ -215,6 +215,7 @@ const Categories = [ "Extract dates", "Regular expression", "XPath expression", + "JPath expression", "CSS selector", "Extract EXIF", ] @@ -278,6 +279,7 @@ const Categories = [ "CSS Beautify", "CSS Minify", "XPath expression", + "JPath expression", "CSS selector", "Strip HTML tags", "Diff", diff --git a/src/core/config/OperationConfig.js b/src/core/config/OperationConfig.js index b0c005b3..38aa35cd 100755 --- a/src/core/config/OperationConfig.js +++ b/src/core/config/OperationConfig.js @@ -2243,6 +2243,24 @@ const OperationConfig = { } ] }, + "JPath expression": { + description: "Extract information from a JSON object with a JPath query.", + run: Code.runJpath, + inputType: "string", + outputType: "string", + args: [ + { + name: "Query", + type: "string", + value: Code.JPATH_INITIAL + }, + { + name: "Result delimiter", + type: "binaryShortString", + value: Code.JPATH_DELIMITER + } + ] + }, "CSS selector": { description: "Extract information from an HTML document with a CSS selector", run: Code.runCSSQuery, diff --git a/src/core/operations/Code.js b/src/core/operations/Code.js index 9840797d..fb4a7e9d 100755 --- a/src/core/operations/Code.js +++ b/src/core/operations/Code.js @@ -4,6 +4,7 @@ import Utils from "../Utils.js"; import vkbeautify from "vkbeautify"; import {DOMParser as dom} from "xmldom"; import xpath from "xpath"; +import jpath from "jsonpath"; import prettyPrintOne from "imports-loader?window=>global!exports-loader?prettyPrintOne!google-code-prettify/bin/prettify.min.js"; @@ -355,6 +356,48 @@ const Code = { }, + /** + * @constant + * @default + */ + JPATH_INITIAL: "", + + /** + * @constant + * @default + */ + JPATH_DELIMITER: "\\n", + + /** + * JPath expression operation. + * + * @author Matt C (matt@artemisbot.uk) + * @param {string} input + * @param {Object[]} args + * @returns {string} + */ + runJpath: function(input, args) { + let query = args[0], + delimiter = args[1], + results, + obj; + + try { + obj = JSON.parse(input); + } catch (err) { + return "Invalid input JSON: " + err.message; + } + + try { + results = jpath.query(obj, query); + } catch (err) { + return "Invalid JPath expression: " + err.message; + } + + return results.map(result => JSON.stringify(result)).join(delimiter); + }, + + /** * @constant * @default diff --git a/test/tests/operations/Code.js b/test/tests/operations/Code.js index 5f6a4329..fee0b5f4 100644 --- a/test/tests/operations/Code.js +++ b/test/tests/operations/Code.js @@ -2,12 +2,54 @@ * Code tests. * * @author tlwr [toby@toby.codes] + * @author Matt C [matt@artemisbot.uk] * * @copyright Crown Copyright 2017 * @license Apache-2.0 */ import TestRegister from "../../TestRegister.js"; +const JPATH_TEST_DATA = { + "store": { + "book": [{ + "category": "reference", + "author": "Nigel Rees", + "title": "Sayings of the Century", + "price": 8.95 + }, { + "category": "fiction", + "author": "Evelyn Waugh", + "title": "Sword of Honour", + "price": 12.99 + }, { + "category": "fiction", + "author": "Herman Melville", + "title": "Moby Dick", + "isbn": "0-553-21311-3", + "price": 8.99 + }, { + "category": "fiction", + "author": "J. R. R. Tolkien", + "title": "The Lord of the Rings", + "isbn": "0-395-19395-8", + "price": 22.99 + }], + "bicycle": { + "color": "red", + "price": 19.95 + }, + "newspaper": [{ + "format": "broadsheet", + "title": "Financial Times", + "price": 2.75 + }, { + "format": "tabloid", + "title": "The Guardian", + "price": 2.00 + }] + } +}; + TestRegister.addTests([ { name: "To Camel case (dumb)", @@ -129,4 +171,143 @@ TestRegister.addTests([ } ], }, + { + name: "JPath Expression: Empty JSON", + input: "", + expectedOutput: "Invalid input JSON: Unexpected end of JSON input", + recipeConfig: [ + { + "op": "JPath expression", + "args": ["", "\n"] + } + ], + }, + { + name: "JPath Expression: Empty expression", + input: JSON.stringify(JPATH_TEST_DATA), + expectedOutput: "Invalid JPath expression: we need a path", + recipeConfig: [ + { + "op": "JPath expression", + "args": ["", "\n"] + } + ], + }, + { + name: "JPath Expression: Fetch of values from specific object", + input: JSON.stringify(JPATH_TEST_DATA), + expectedOutput: [ + "\"Nigel Rees\"", + "\"Evelyn Waugh\"", + "\"Herman Melville\"", + "\"J. R. R. Tolkien\"" + ].join("\n"), + recipeConfig: [ + { + "op": "JPath expression", + "args": ["$.store.book[*].author", "\n"] + } + ], + }, + { + name: "JPath Expression: Fetch of all values with matching key", + input: JSON.stringify(JPATH_TEST_DATA), + expectedOutput: [ + "\"Sayings of the Century\"", + "\"Sword of Honour\"", + "\"Moby Dick\"", + "\"The Lord of the Rings\"", + "\"Financial Times\"", + "\"The Guardian\"" + ].join("\n"), + recipeConfig: [ + { + "op": "JPath expression", + "args": ["$..title", "\n"] + } + ], + }, + { + name: "JPath Expression: All data in object", + input: JSON.stringify(JPATH_TEST_DATA), + expectedOutput: [ + "[{\"category\":\"reference\",\"author\":\"Nigel Rees\",\"title\":\"Sayings of the Century\",\"price\":8.95},{\"category\":\"fiction\",\"author\":\"Evelyn Waugh\",\"title\":\"Sword of Honour\",\"price\":12.99},{\"category\":\"fiction\",\"author\":\"Herman Melville\",\"title\":\"Moby Dick\",\"isbn\":\"0-553-21311-3\",\"price\":8.99},{\"category\":\"fiction\",\"author\":\"J. R. R. Tolkien\",\"title\":\"The Lord of the Rings\",\"isbn\":\"0-395-19395-8\",\"price\":22.99}]", + "{\"color\":\"red\",\"price\":19.95}", + "[{\"format\":\"broadsheet\",\"title\":\"Financial Times\",\"price\":2.75},{\"format\":\"tabloid\",\"title\":\"The Guardian\",\"price\":2}]" + ].join("\n"), + recipeConfig: [ + { + "op": "JPath expression", + "args": ["$.store.*", "\n"] + } + ], + }, + { + name: "JPath Expression: Last element in array", + input: JSON.stringify(JPATH_TEST_DATA), + expectedOutput: "{\"category\":\"fiction\",\"author\":\"J. R. R. Tolkien\",\"title\":\"The Lord of the Rings\",\"isbn\":\"0-395-19395-8\",\"price\":22.99}", + recipeConfig: [ + { + "op": "JPath expression", + "args": ["$..book[-1:]", "\n"] + } + ], + }, + { + name: "JPath Expression: First 2 elements in array", + input: JSON.stringify(JPATH_TEST_DATA), + expectedOutput: [ + "{\"category\":\"reference\",\"author\":\"Nigel Rees\",\"title\":\"Sayings of the Century\",\"price\":8.95}", + "{\"category\":\"fiction\",\"author\":\"Evelyn Waugh\",\"title\":\"Sword of Honour\",\"price\":12.99}" + ].join("\n"), + recipeConfig: [ + { + "op": "JPath expression", + "args": ["$..book[:2]", "\n"] + } + ], + }, + { + name: "JPath Expression: All elements in array with property", + input: JSON.stringify(JPATH_TEST_DATA), + expectedOutput: [ + "{\"category\":\"fiction\",\"author\":\"Herman Melville\",\"title\":\"Moby Dick\",\"isbn\":\"0-553-21311-3\",\"price\":8.99}", + "{\"category\":\"fiction\",\"author\":\"J. R. R. Tolkien\",\"title\":\"The Lord of the Rings\",\"isbn\":\"0-395-19395-8\",\"price\":22.99}" + ].join("\n"), + recipeConfig: [ + { + "op": "JPath expression", + "args": ["$..book[?(@.isbn)]", "\n"] + } + ], + }, + { + name: "JPath Expression: All elements in array which meet condition", + input: JSON.stringify(JPATH_TEST_DATA), + expectedOutput: [ + "{\"category\":\"fiction\",\"author\":\"Evelyn Waugh\",\"title\":\"Sword of Honour\",\"price\":12.99}", + "{\"category\":\"fiction\",\"author\":\"Herman Melville\",\"title\":\"Moby Dick\",\"isbn\":\"0-553-21311-3\",\"price\":8.99}", + "{\"category\":\"fiction\",\"author\":\"J. R. R. Tolkien\",\"title\":\"The Lord of the Rings\",\"isbn\":\"0-395-19395-8\",\"price\":22.99}" + ].join("\n"), + recipeConfig: [ + { + "op": "JPath expression", + "args": ["$..book[?(@.price<30 && @.category==\"fiction\")]", "\n"] + } + ], + }, + { + name: "JPath Expression: All elements in object", + input: JSON.stringify(JPATH_TEST_DATA), + expectedOutput: [ + "{\"category\":\"reference\",\"author\":\"Nigel Rees\",\"title\":\"Sayings of the Century\",\"price\":8.95}", + "{\"category\":\"fiction\",\"author\":\"Herman Melville\",\"title\":\"Moby Dick\",\"isbn\":\"0-553-21311-3\",\"price\":8.99}" + ].join("\n"), + recipeConfig: [ + { + "op": "JPath expression", + "args": ["$..book[?(@.price<10)]", "\n"] + } + ], + }, ]);