Made all obvious variables user-configurable

There's a bit of refactoring in order for the webextension to deal with
the new order of initialisation now that config is sent by the Golang
client.

Closes #83

interfacer/src/browsh/browsh.go

@@ -16,8 +16,8 @@ import (
 	"github.com/gdamore/tcell"
 
 	"github.com/go-errors/errors"
-	"github.com/spf13/viper"
 	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
 )
 
 var (
@@ -77,7 +77,8 @@ func Log(msg string) {
 	}
 }
 
-func initialise() {
+// Initialise browsh
+func Initialise() {
 	if IsTesting {
 		*isDebug = true
 	}
@@ -145,7 +146,6 @@ func Shell(command string) string {
 // TTYStart starts Browsh
 func TTYStart(injectedScreen tcell.Screen) {
 	screen = injectedScreen
-	initialise()
 	setupTcell()
 	writeString(1, 0, logo, tcell.StyleDefault)
 	writeString(0, 15, "Starting Browsh, the modern text-based web browser.", tcell.StyleDefault)
@@ -190,6 +190,7 @@ func ttyEntry() {
 // MainEntry decides between running Browsh as a CLI app or as an HTTP web server
 func MainEntry() {
 	pflag.Parse()
+	Initialise()
 	if viper.GetBool("http-server-mode") {
 		HTTPServerStart()
 	} else {

interfacer/src/browsh/comms.go

@@ -141,6 +141,7 @@ func webSocketServer(w http.ResponseWriter, r *http.Request) {
 	isConnectedToWebExtension = true
 	go webSocketWriter(ws)
 	go webSocketReader(ws)
+	sendConfigToWebExtension()
 	if viper.GetBool("http-server-mode") {
 		sendMessageToWebExtension("/raw_text_mode")
 	} else {
@@ -150,3 +151,8 @@ func webSocketServer(w http.ResponseWriter, r *http.Request) {
 	// work. So we do it here instead.
 	sendMessageToWebExtension("/new_tab," + viper.GetString("startup-url"))
 }
+
+func sendConfigToWebExtension() {
+	configJSON, _ := json.Marshal(viper.AllSettings())
+	sendMessageToWebExtension("/config," + string(configJSON))
+}

interfacer/src/browsh/config.go

@@ -1,11 +1,11 @@
 package browsh
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
-	"bytes"
 
 	"github.com/shibukawa/configdir"
 	"github.com/spf13/pflag"
@@ -80,14 +80,12 @@ func loadConfig() {
 	// First load the sample config in case the user hasn't updated any new fields
 	err := viper.ReadConfig(bytes.NewBuffer([]byte(configSample)))
 	if err != nil {
-		Shutdown(err)
+		panic(fmt.Errorf("Config file error: %s \n", err))
 	}
 	// Then load the users own config file, overwriting the sample config
 	err = viper.MergeInConfig()
 	if err != nil {
-		Shutdown(err)
+		panic(fmt.Errorf("Config file error: %s \n", err))
 	}
 	viper.BindPFlags(pflag.CommandLine)
-	Log("Using the folowing config values:")
-	Log(fmt.Sprintf("%v", viper.AllSettings()))
 }

interfacer/src/browsh/config_sample.go

@@ -1,7 +1,17 @@
 package browsh
 
-var configSample =
+var configSample = `
-`[browsh]
+# See; https://www.brow.sh/donate/
+# By showing your support you can disable the app's branding and nags to donate.
+browsh_supporter = "♥"
+
+# The base query when a non-URL is entered into the URL bar
+default_search_engine_base = "https://www.google.com/search?q="
+
+# The mobile user agent for forcing web pages to use their mobile layout
+mobile_user_agent = "Mozilla/5.0 (Android 7.0; Mobile; rv:54.0) Gecko/58.0 Firefox/58.0"
+
+[browsh] # Browsh internals
 websocket-port = 3334
 
 [firefox]
@@ -10,7 +20,37 @@ profile = "default"
 use-existing = false
 with-gui = false
 
+[tty]
+# The time in milliseconds between requesting a new TTY-sized pixel frame.
+# This is essentially the frame rate for graphics. Lower values make for smoother
+# animations and feedback, but also increases the CPU load.
+small_pixel_frame_rate = 250
+
 [http-server]
 port = 4333
 bind = "0.0.0.0"
+
+# The time to wait in milliseconds after the DOM is ready before
+# trying to parse and render the page's text. Too soon and text risks not being
+# parsed, too long and you wait unecessarily.
+render_delay = 400
+
+# The dimensions of a char-based window onto a webpage.
+# The columns are ultimately the width of the final text. Whereas the rows
+# represent the height of the original web page made visible to the original
+# browser window. So the number of rows can effect things like how far down a
+# web page images are lazy-loaded.
+columns = 100
+rows = 30
+
+# The amount of lossy JPG compression to apply to the background image of HTML
+# pages.
+jpeg_compression = 0.9
+
+# Rate limit. For syntax, see: https://github.com/ulule/limiter
+rate-limit = "10-M"
+
+# HTML snippets to show at top and bottom of final page.
+header = ""
+footer = ""
 `

interfacer/src/browsh/raw_text_server.go

@@ -10,8 +10,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/spf13/viper"
 	"github.com/NYTimes/gziphandler"
+	"github.com/spf13/viper"
 	"github.com/ulule/limiter"
 	"github.com/ulule/limiter/drivers/middleware/stdlib"
 	"github.com/ulule/limiter/drivers/store/memory"
@@ -28,7 +28,6 @@ var rawTextRequests = make(map[string]string)
 // it will return:
 // `Something                                                                    `
 func HTTPServerStart() {
-	initialise()
 	startFirefox()
 	go startWebSocketServer()
 	Log("Starting Browsh HTTP server")
@@ -44,9 +43,9 @@ func HTTPServerStart() {
 }
 
 func setupRateLimiter() *stdlib.Middleware {
-	rate := limiter.Rate{
+	rate, err := limiter.NewRateFromFormatted(viper.GetString("http-server.rate-limit"))
-		Period: 1 * time.Minute,
+	if err != nil {
-		Limit:  10,
+		Shutdown(err)
 	}
 	// TODO: Centralise store amongst instances with Redis
 	store := memory.NewStore()

interfacer/src/browsh/ui.go

@@ -1,8 +1,8 @@
 package browsh
 
 import (
-	"github.com/spf13/viper"
 	"github.com/gdamore/tcell"
+	"github.com/spf13/viper"
 )
 
 var (

interfacer/test/http-server/setup.go

@@ -8,8 +8,8 @@ import (
 
 	ginkgo "github.com/onsi/ginkgo"
 
-	"github.com/spf13/viper"
 	"browsh/interfacer/src/browsh"
+	"github.com/spf13/viper"
 )
 
 var staticFileServerPort = "4444"
@@ -23,6 +23,7 @@ func startStaticFileServer() {
 
 func startBrowsh() {
 	browsh.IsTesting = true
+	browsh.Initialise()
 	viper.Set("http-server-mode", true)
 	browsh.HTTPServerStart()
 }

interfacer/test/tty/setup.go

@@ -185,6 +185,7 @@ func startStaticFileServer() {
 func startBrowsh() {
 	browsh.IsTesting = true
 	simScreen = tcell.NewSimulationScreen("UTF-8")
+	browsh.Initialise()
 	browsh.TTYStart(simScreen)
 }
 

webext/contrib/bundle_webextension.sh

@@ -13,7 +13,8 @@ NODE_BIN=$PROJECT_ROOT/webext/node_modules/.bin
 destination=$PROJECT_ROOT/interfacer/src/browsh/webextension.go
 
 cd $PROJECT_ROOT/webext && $NODE_BIN/webpack
-cd $PROJECT_ROOT/webext/dist && $NODE_BIN/web-ext build --overwrite-dest
+cd $PROJECT_ROOT/webext/dist && rm *.map
+$NODE_BIN/web-ext build --overwrite-dest
 
 # Get the current version of Browsh
 version=$(cat $PROJECT_ROOT/webext/manifest.json | python2 -c \

webext/src/background/dimensions.js

@@ -16,16 +16,11 @@ export default class extends utils.mixins(CommonMixin) {
     // actually see a little bit of white at the bottom perhaps from where the screen capture
     // goes over the bottom of the viewport.
     this._window_ui_magic_number = 3;
-    // The Browsh HTTP Server service doesn't load a TTY, so we need to supply the size.
+  }
-    // Strictly it shouldn't even be needed if the code was completely refactored. Although
+
-    // it should be worth taking into consideration how the size of the TTY and therefore the
+  postConfigSetup(config) {
-    // resized browser window affects the rendering of a web page, for instance images outside
+    this.config = config;
-    // of the viewport can sometimes not be loaded. So is it practical to set the TTY size to
+    this._setRawTextTTYSize();
-    // the size of the entire DOM?
-    this.raw_text_tty_size = {
-      width: 100,
-      height: 30
-    };
   }
 
   setCharValues(incoming) {
@@ -42,6 +37,19 @@ export default class extends utils.mixins(CommonMixin) {
     }
   }
 
+  // The Browsh HTTP Server service doesn't load a TTY, so we need to supply the size.
+  // Strictly it shouldn't even be needed if the code was completely refactored. Although
+  // it should be worth taking into consideration how the size of the TTY and therefore the
+  // resized browser window affects the rendering of a web page, for instance images outside
+  // of the viewport can sometimes not be loaded. So is it practical to set the TTY size to
+  // the size of the entire DOM?
+  _setRawTextTTYSize() {
+    this.raw_text_tty_size = {
+      width: this.config["http-server"].columns,
+      height: this.config["http-server"].rows
+    };
+  }
+
   resizeBrowserWindow() {
     if (
       !this.tty.width ||

webext/src/background/manager.js

@@ -21,14 +21,10 @@ export default class extends utils.mixins(CommonMixin, TTYCommandsMixin) {
     // Used so that reconnections to the terminal don't also attempt to reconnect to the
     // browser DOM.
     this._is_connected_to_browser_dom = false;
-    // The time in milliseconds between requesting a new TTY-size pixel frame
-    this._small_pixel_frame_rate = 250;
     // Raw text mode is for when Browsh is running as an HTTP server that serves single
     // pages as entire DOMs, in plain text.
     this._is_raw_text_mode = false;
-    // A mobile user agent for forcing web pages to use its mobile layout
+    // Toggle user agent
-    this._mobile_user_agent =
-      "Mozilla/5.0 (Android 7.0; Mobile; rv:54.0) Gecko/58.0 Firefox/58.0";
     this._is_using_mobile_user_agent = false;
     this._addUserAgentListener();
     // Listen to HTTP requests. This allows us to display some helpful status messages at the
@@ -48,7 +44,6 @@ export default class extends utils.mixins(CommonMixin, TTYCommandsMixin) {
       this.dimensions.terminal = this.terminal;
       this._listenForTerminalMessages();
       this._connectToBrowserDOM();
-      this._startFrameRequestLoop();
     });
     this.terminal.addEventListener("close", _event => {
       this._reconnectToTerminal();
@@ -139,6 +134,7 @@ export default class extends utils.mixins(CommonMixin, TTYCommandsMixin) {
     let tab = this.tabs[native_tab_object.id];
     tab.native_last_change = changes;
     tab.ensureConnectionToBackground();
+    tab.sendGlobalConfig(this.config);
   }
 
   // Note that although this callback signifies that the tab now exists, it is not fully
@@ -163,11 +159,13 @@ export default class extends utils.mixins(CommonMixin, TTYCommandsMixin) {
 
   _applyUpdates(tabish_object) {
     let tab = this._maybeNewTab({ id: tabish_object.id });
-    ["id", "title", "url", "active", "request_id"].map(key => {
+    ["id", "title", "url", "active", "request_id", "raw_text_mode_type"].map(
+      key => {
         if (tabish_object.hasOwnProperty(key)) {
           tab[key] = tabish_object[key];
         }
-    });
+      }
+    );
     if (tabish_object.active) {
       this.active_tab_id = tab.id;
     }
@@ -186,7 +184,10 @@ export default class extends utils.mixins(CommonMixin, TTYCommandsMixin) {
       `Tab ${channel.name} connected for communication with background process`
     );
     let tab = this.tabs[parseInt(channel.name)];
-    tab.postConnectionInit(channel);
+    tab.postConnectionInit(channel, this.config);
+    if (!this._is_connected_to_browser_dom) {
+      this._startFrameRequestLoop();
+    }
     this._is_connected_to_browser_dom = true;
   }
 
@@ -240,13 +241,17 @@ export default class extends utils.mixins(CommonMixin, TTYCommandsMixin) {
   // graphics pixles are sent. Larger frames are sent in response to scroll events and
   // TTY-sized text frames are sent in response to DOM mutation events.
   _startFrameRequestLoop() {
-    this.log("BACKGROUND: Frame loop starting");
+    this.log(
+      "BACKGROUND: Frame loop starting at " +
+        this.config.tty.small_pixel_frame_rate +
+        "ms intervals"
+    );
     setInterval(() => {
       if (this._is_initial_window_size_pending) this._initialWindowResize();
       if (this._isAbleToRequestFrame()) {
         this.sendToCurrentTab("/request_frame");
       }
-    }, this._small_pixel_frame_rate);
+    }, this.config.tty.small_pixel_frame_rate);
   }
 
   _isAbleToRequestFrame() {

webext/src/background/tab.js

@@ -20,10 +20,11 @@ export default class extends utils.mixins(CommonMixin, TabCommandsMixin) {
     this._closeUnwantedStartupTabs();
   }
 
-  postConnectionInit(channel) {
+  postConnectionInit(channel, config) {
     this.channel = channel;
     this._sendTTYDimensions();
     this._listenForMessages();
+    this.sendGlobalConfig(config);
     this._calculateMode();
   }
 
@@ -32,7 +33,7 @@ export default class extends utils.mixins(CommonMixin, TabCommandsMixin) {
     if (!this._is_raw_text_mode) {
       mode = "interactive";
     } else {
-      mode = this.raw_text_mode_type;
+      mode = "raw_text_" + this.raw_text_mode_type;
     }
     this.channel.postMessage(`/mode,${mode}`);
   }
@@ -118,13 +119,8 @@ export default class extends utils.mixins(CommonMixin, TabCommandsMixin) {
     }
   }
 
-  setMode(mode) {
+  sendGlobalConfig(config) {
-    this.raw_text_mode_type = mode;
+    this.channel.postMessage(`/config,${JSON.stringify(config)}`);
-    // Send it here, in case there is a race condition with the postCommsInit() not knowing
-    // the mode.
-    if (this._is_raw_text_mode) {
-      this.channel.postMessage(`/mode,${mode}`);
-    }
   }
 
   _listenForMessages() {

webext/src/background/tty_commands_mixin.js

@@ -7,6 +7,9 @@ export default MixinBase =>
       const parts = message.split(",");
       const command = parts[0];
       switch (command) {
+        case "/config":
+          this._loadConfig(message.slice(8));
+          break;
         case "/tab_command":
           this.sendToCurrentTab(message.slice(13));
           break;
@@ -42,6 +45,15 @@ export default MixinBase =>
       }
     }
 
+    _loadConfig(json_string) {
+      this.log(json_string);
+      this.config = JSON.parse(json_string);
+      if (this.currentTab()) {
+        this.currentTab().sendGlobalConfig(this.config);
+      }
+      this.dimensions.postConfigSetup(this.config);
+    }
+
     _updateTTYSize(width, height) {
       this.dimensions.tty.width = parseInt(width);
       this.dimensions.tty.height = parseInt(height);
@@ -87,7 +99,7 @@ export default MixinBase =>
     // TODO: move to CLI client
     _getURLfromUserInput(input) {
       let url;
-      const search_engine = "https://www.google.com/search?q=";
+      const search_engine = this.config.default_search_engine_base;
       // Basically just check to see if there is text either side of a dot
       const is_straddled_dot = RegExp(/^[^\s]+\.[^\s]+/);
       // More comprehensive URL pattern
@@ -171,9 +183,9 @@ export default MixinBase =>
       this.createNewTab(url, native_tab => {
         this._acknowledgeNewTab({
           id: native_tab.id,
-          request_id: request_id
+          request_id: request_id,
+          raw_text_mode_type: mode.toLowerCase()
         });
-        this.tabs[native_tab.id].setMode(`raw_text_${mode.toLowerCase()}`);
       });
     }
 
@@ -194,7 +206,7 @@ export default MixinBase =>
           if (this._is_using_mobile_user_agent) {
             e.requestHeaders.forEach(header => {
               if (header.name.toLowerCase() == "user-agent") {
-                header.value = this._mobile_user_agent;
+                header.value = this.config.mobile_user_agent;
               }
             });
             return { requestHeaders: e.requestHeaders };

webext/src/dom/commands_mixin.js

@@ -3,13 +3,17 @@ import utils from "utils";
 export default MixinBase =>
   class extends MixinBase {
     _handleBackgroundMessage(message) {
-      let input, url;
+      let input, url, config;
       const parts = message.split(",");
       const command = parts[0];
       switch (command) {
         case "/mode":
           this._setupMode(parts[1]);
           break;
+        case "/config":
+          config = JSON.parse(utils.rebuildArgsToSingleArg(parts));
+          this._loadConfig(config);
+          break;
         case "/request_frame":
           this.sendFrame();
           break;
@@ -61,6 +65,11 @@ export default MixinBase =>
       }
     }
 
+    _loadConfig(config) {
+      this.config = config;
+      this._postSetupConstructor();
+    }
+
     _handleUserInput(input) {
       this._handleSpecialKeys(input);
       this._handleCharBasedKeys(input);

webext/src/dom/graphics_builder.js

@@ -7,13 +7,12 @@ import CommonMixin from "dom/common_mixin";
 // to aid in a clean separation of the graphics and text in the final frame
 // rendered in the terminal.
 export default class extends utils.mixins(CommonMixin) {
-  constructor(channel, dimensions) {
+  constructor(channel, dimensions, config) {
     super();
     this.channel = channel;
     this.dimensions = dimensions;
-    // The amount of lossy JPG compression to apply to the HTML services
+    this.config = config;
-    // background image
+    this._html_image_compression = this.config["http-server"].jpeg_compression;
-    this._html_image_compression = 0.9;
     this._screenshot_canvas = document.createElement("canvas");
     this._converter_canvas = document.createElement("canvas");
     this._screenshot_ctx = this._screenshot_canvas.getContext("2d");

webext/src/dom/manager.js

@@ -20,13 +20,18 @@ export default class extends utils.mixins(CommonMixin, CommandsMixin) {
     this._setupInit();
   }
 
-  _postCommsConstructor() {
+  _postSetupConstructor() {
     this.dimensions.channel = this.channel;
-    this.graphics_builder = new GraphicsBuilder(this.channel, this.dimensions);
+    this.graphics_builder = new GraphicsBuilder(
+      this.channel,
+      this.dimensions,
+      this.config
+    );
     this.text_builder = new TextBuilder(
       this.channel,
       this.dimensions,
-      this.graphics_builder
+      this.graphics_builder,
+      this.config
     );
     this.text_builder._raw_text_start = performance.now();
   }
@@ -96,7 +101,6 @@ export default class extends utils.mixins(CommonMixin, CommandsMixin) {
 
   _postCommsInit() {
     this.log("Webextension postCommsInit()");
-    this._postCommsConstructor();
     this._sendTabInfo();
     this.sendMessage("/status,page_init");
     this._listenForBackgroundMessages();
@@ -108,7 +112,6 @@ export default class extends utils.mixins(CommonMixin, CommandsMixin) {
     this._setupDebouncedFunctions();
     this._startMutationObserver();
     this.sendAllBigFrames();
-    // Send again for pages that have page load transition effects :/
     // TODO:
     //   Disabling CSS transitions is not easy, many pages won't even render
     //   if they're disabled. Eg; Google's login process.

webext/src/dom/serialise_mixin.js

@@ -48,24 +48,53 @@ export default MixinBase =>
     }
 
     _wrapHTML(raw_text) {
-      let info = "";
+      return this._getHTMLHead() + raw_text + this._getFooter();
-      const head = this._getHTMLHead();
-      const date_time = this._getCurrentDataTime();
-      const elapsed = `${performance.now() - this._raw_text_start}ms`;
-      info +=
-        "\n\n" +
-        `Built by <a href="https://www.brow.sh">Browsh</a> ` +
-        `on ${date_time} in ${elapsed}.`;
-      if (this.dimensions.is_page_truncated) {
-        info +=
-          "\nBrowsh parser: the page was too large, some text may have been truncated.";
     }
 
-      const donate =
+    // Whether a use has shown support. This controls certain Browsh branding and
-        '\nPlease consider <a href="https://www.brow.sh/donate/">donating</a> ' +
+    // nags to donate.
-        "to help all those with slow and/or expensive internet.";
+    userHasShownSupport() {
-      const foot = `<span class="browsh-footer">${info}${donate}</span></pre></body></html>`;
+      this.log(this.config.browsh_supporter);
-      return head + raw_text + foot;
+      return (
+        this.config.browsh_supporter === "I have shown my support for Browsh"
+      );
+    }
+
+    _byBrowsh() {
+      if (this.userHasShownSupport()) {
+        return "";
+      }
+      return 'by <a href="https://www.brow.sh">Browsh</a> ';
+    }
+
+    _getUserFooter() {
+      return "\n" + this.config["http-server"].footer;
+    }
+
+    _getUserHeader() {
+      return this.config["http-server"].header + "\n";
+    }
+
+    _getMetaData() {
+      let metadata = "";
+      const date_time = this._getCurrentDataTime();
+      const elapsed = `${performance.now() - this._raw_text_start}ms`;
+      metadata +=
+        "\n\n" + `Built ` + this._byBrowsh() + `on ${date_time} in ${elapsed}.`;
+      if (this.dimensions.is_page_truncated) {
+        metadata +=
+          "\nBrowsh parser: the page was too large, some text may have been truncated.";
+      }
+      return metadata;
+    }
+
+    _getFooter() {
+      return (
+        '<span class="browsh-footer">' +
+        this._getMetaData() +
+        this._getUserFooter() +
+        `</span></pre></body></html>`
+      );
     }
 
     _getHTMLHead() {
@@ -111,6 +140,7 @@ export default MixinBase =>
        </style>
      </head>
      <body>
+     ${this._getUserHeader()}
      <pre>`;
     }
 

webext/src/dom/text_builder.js

@@ -9,11 +9,12 @@ import TTYGrid from "dom/tty_grid";
 // Convert the text on the page into a snapped 2-dimensional grid to be displayed directly
 // in the terminal.
 export default class extends utils.mixins(CommonMixin, SerialiseMixin) {
-  constructor(channel, dimensions, graphics_builder) {
+  constructor(channel, dimensions, graphics_builder, config) {
     super();
     this.channel = channel;
     this.dimensions = dimensions;
     this.graphics_builder = graphics_builder;
+    this.config = config;
     this.tty_grid = new TTYGrid(dimensions, graphics_builder);
     this._parse_started_elements = [];
     // A `range` is the DOM's representation of elements and nodes as they are rendered in
@@ -37,7 +38,7 @@ export default class extends utils.mixins(CommonMixin, SerialiseMixin) {
     setTimeout(() => {
       this.buildFormattedText();
       this._sendRawText();
-    }, 400);
+    }, this.config["http-server"].render_delay);
   }
 
   buildFormattedText() {

webext/webpack.config.js

@@ -3,6 +3,7 @@ const path = require('path');
 const CopyWebpackPlugin = require('copy-webpack-plugin');
 
 module.exports = {
+  mode: process.env['BROWSH_ENV'] === 'RELEASE' ? 'production' : 'development',
   target: 'node',
   entry: {
     content: './content.js',
@@ -18,6 +19,7 @@ module.exports = {
       'node_modules'
     ]
   },
+  devtool: 'source-map',
   plugins: [
     new webpack.DefinePlugin({
       DEVELOPMENT: JSON.stringify(true),