Add anchor tags to HTTP Server output

This means you can now load the raw text in a browser and the resulting
page will have basic blue links that can be clicked on that will in turn
be loaded by the HTTP service.

A significant feature, so worthy of a minor version bump to;
v1.1.0

interfacer/src/browsh/raw_text_server.go

@@ -60,10 +60,23 @@ func (h *slashFix) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 func handleHTTPServerRequest(w http.ResponseWriter, r *http.Request) {
 	urlForBrowsh, _ := url.PathUnescape(strings.TrimPrefix(r.URL.Path, "/"))
 	rawTextRequestID := pseudoUUID()
-	sendMessageToWebExtension("/raw_text_request," + rawTextRequestID + "," + urlForBrowsh)
+	mode := getRawTextMode(r)
+	sendMessageToWebExtension(
+		"/raw_text_request," + rawTextRequestID + "," +
+		mode + "," +
+		urlForBrowsh)
 	waitForResponse(rawTextRequestID, w)
 }
 
+// 'PLAIN' mode returns raw text without any HTML whatsoever.
+// 'HTML' mode returns some basic HTML tags for things like anchor links.
+func getRawTextMode(r *http.Request) string {
+	var mode = "HTML"
+	if (strings.Contains(r.Host, "text.")) { mode = "PLAIN" }
+	if (r.Header.Get("X-Browsh-Raw-Mode") == "PLAIN") { mode = "PLAIN" }
+	return mode
+}
+
 func waitForResponse(rawTextRequestID string, w http.ResponseWriter) {
 	var rawTextRequestResponse string
 	var ok bool

interfacer/test/http-server/server_test.go

@@ -13,9 +13,16 @@ func TestHTTPServer(t *testing.T) {
 }
 
 var _ = Describe("HTTP Server", func() {
-	It("should return text", func() {
-		response := getPath("/smorgasbord")
+	It("should return plain text", func() {
+		response := getPath("/smorgasbord", "plain")
 		Expect(response).To(ContainSubstring("multiple hot       Smörgås"))
 		Expect(response).To(ContainSubstring("A special Swedish type of smörgåsbord"))
+		Expect(response).ToNot(ContainSubstring("<a href"))
+	})
+
+	It("should return HTML text", func() {
+		response := getPath("/smorgasbord", "html")
+		Expect(response).To(ContainSubstring(
+			"<a href=\"/http://localhost:4444/smorgasbord/another.html\">Another page</a>"))
 	})
 })

interfacer/test/http-server/setup.go

@@ -26,11 +26,16 @@ func startBrowsh() {
 	browsh.HTTPServerStart()
 }
 
-func getPath(path string) string {
+func getPath(path string, mode string) string {
 	browshServiceBase := "http://localhost:" + *browsh.HTTPServerPort
 	staticFileServerBase := "http://localhost:" + staticFileServerPort
 	fullBase := browshServiceBase + "/" + staticFileServerBase
-	response, err := http.Get(fullBase + path)
+	client := &http.Client{}
+	request, err := http.NewRequest("GET", fullBase + path, nil)
+	if mode == "plain" {
+		request.Header.Add("X-Browsh-Raw-Mode", "PLAIN")
+	}
+	response, err := client.Do(request)
 	if err != nil {
 		panic(fmt.Sprintf("%s", err))
 	} else {
@@ -57,7 +62,7 @@ var _ = ginkgo.BeforeSuite(func() {
 	time.Sleep(10 * time.Second)
 	// Allow the browser to sort its sizing out, because sometimes the first test catches the
 	// browser before it's completed its resizing.
-	getPath("/smorgasbord")
+	getPath("/smorgasbord", "plain")
 })
 
 var _	= ginkgo.AfterSuite(func() {

webext/manifest.json

@@ -1,7 +1,7 @@
 {
   "manifest_version": 2,
   "name": "Browsh",
-  "version": "1.0.13",
+  "version": "1.1.0",
 
   "description": "Renders the browser as realtime, interactive, TTY-compatible text",
 

webext/src/background/manager.js

@@ -176,7 +176,6 @@ export default class extends utils.mixins(CommonMixin, TTYCommandsMixin) {
     this.log(`Tab ${channel.name} connected for communication with background process`);
     let tab = this.tabs[parseInt(channel.name)];
     tab.postConnectionInit(channel);
-    tab.setMode(this._is_raw_text_mode);
     this._is_connected_to_browser_dom = true;
   }
 

webext/src/background/tab.js

@@ -10,6 +10,8 @@ export default class extends utils.mixins(CommonMixin, TabCommandsMixin) {
     this._tab_reloads = 0;
     // The maximum amount of times to try to recover a tab that won't connect
     this._max_number_of_tab_recovery_reloads = 3;
+    // Type of raw text mode; HTML or plain
+    this.raw_text_mode = '';
   }
 
   postDOMLoadInit(terminal, dimensions) {
@@ -22,6 +24,9 @@ export default class extends utils.mixins(CommonMixin, TabCommandsMixin) {
     this.channel = channel;
     this._sendTTYDimensions();
     this._listenForMessages();
+    let mode = 'interactive';
+    if (this.raw_text_mode !== '') { mode = this.raw_text_mode }
+    this.channel.postMessage(`/mode,${mode}`);
   }
 
   isConnected() {
@@ -104,26 +109,8 @@ export default class extends utils.mixins(CommonMixin, TabCommandsMixin) {
     }
   }
 
-  setMode(is_raw_text_mode) {
-    if (is_raw_text_mode) {
-      this.channel.postMessage('/mode,raw_text');
-      this._requestRawText();
-    } else {
-      this.channel.postMessage('/mode,interactive');
-    }
-  }
-
-  // If Browsh is setup in HTTP-server mode then this is the moment that we ask the tab to
-  // render the entire DOM as plain text. We must only do this for tabs subsequent to the
-  // initial tab that loads at boot time (there must always remain a single tab to keep the
-  // browser running).
-  _requestRawText() {
-    // The assumption is that Tab ID 1 is always the first tab. However, I have a vague
-    // memory of seeing the "Firefox Privacy Notice" tab load before the CLI argument requested
-    // URL. So, maybe ID 1 isn't 100% reliable.
-    if (this.id !== 1) {
-      this.channel.postMessage('/request_raw_text');
-    }
+  setMode(mode) {
+    this.raw_text_mode = mode;
   }
 
   _listenForMessages() {
@@ -146,6 +133,7 @@ export default class extends utils.mixins(CommonMixin, TabCommandsMixin) {
   // first. So let's just close that tab.
   // TODO: Only do this for a testing ENV?
   _closeUnwantedStartupTabs() {
+    if (this.title === undefined) { return false }
     if (
       this.title.includes('Firefox by default shares data to:') ||
       this.title.includes('Firefox Privacy Notice')

webext/src/background/tty_commands_mixin.js

@@ -36,7 +36,7 @@ export default (MixinBase) => class extends MixinBase {
         );
         break;
       case '/raw_text_request':
-        this._rawTextRequest(parts[1], parts.slice(2).join(','));
+        this._rawTextRequest(parts[1], parts[2], parts.slice(3).join(','));
         break;
     }
   }
@@ -164,12 +164,13 @@ export default (MixinBase) => class extends MixinBase {
     this.sendToTerminal('/screenshot,' + data);
   }
 
-  _rawTextRequest(request_id, url) {
-    this.createNewTab(url, tab => {
+  _rawTextRequest(request_id, mode, url) {
+    this.createNewTab(url, native_tab => {
       this._acknowledgeNewTab({
-        id: tab.id,
+        id: native_tab.id,
         request_id: request_id
-      })
+      });
+      this.tabs[native_tab.id].setMode(`raw_text_${mode.toLowerCase()}`);
     });
   }
 

webext/src/dom/commands_mixin.js

@@ -17,9 +17,6 @@ export default (MixinBase) => class extends MixinBase {
           this.sendAllBigFrames();
         }
         break;
-      case '/request_raw_text':
-        this.sendRawText();
-        break;
       case '/scroll_status':
         this._handleScroll(parts[1], parts[2]);
         break;
@@ -50,10 +47,14 @@ export default (MixinBase) => class extends MixinBase {
   }
 
   _setupMode(mode) {
-    if (mode === 'raw_text') {
+    if (mode === 'raw_text_plain' || mode === 'raw_text_html') {
       this._is_raw_text_mode = true;
+      this._is_interactive_mode = false;
+      this._raw_mode_type = mode;
+      this.sendRawText();
     }
     if (mode === 'interactive') {
+      this._is_raw_text_mode = false;
       this._is_interactive_mode = true;
       this._setupInteractiveMode();
     }

webext/src/dom/manager.js

@@ -16,7 +16,7 @@ export default class extends utils.mixins(CommonMixin, CommandsMixin) {
     // For Browsh used via the interactive CLI ap
     this._is_interactive_mode = false;
     // For Browsh used via the HTTP server
-    this._is_raw_text_mode = false;
+    this._is_raw_mode = false;
     this._setupInit();
   }
 
@@ -44,7 +44,7 @@ export default class extends utils.mixins(CommonMixin, CommandsMixin) {
   }
 
   sendAllBigFrames() {
-    if (this._is_raw_text_mode) { return }
+    if (this._is_raw_mode) { return }
     if (!this.dimensions.tty.width) {
       this.log("Not sending big frames without TTY data")
       return
@@ -62,7 +62,7 @@ export default class extends utils.mixins(CommonMixin, CommandsMixin) {
   sendRawText() {
     this.dimensions.update();
     this.dimensions.setSubFrameDimensions('raw_text');
-    this.text_builder.sendRawText();
+    this.text_builder.sendRawText(this._raw_mode_type);
   }
 
   sendSmallPixelFrame() {

webext/src/dom/serialise_mixin.js

@@ -0,0 +1,202 @@
+import utils from 'utils';
+
+export default (MixinBase) => class extends MixinBase {
+  __serialiseFrame() {
+    let cell, index;
+    const top = this.dimensions.frame.sub.top / 2;
+    const left = this.dimensions.frame.sub.left;
+    const bottom = top + (this.dimensions.frame.sub.height / 2);
+    const right = left + this.dimensions.frame.sub.width;
+    this._setupFrameMeta();
+    this._serialiseInputBoxes();
+    for (let y = top; y < bottom; y++) {
+      for (let x = left; x < right; x++) {
+        index = (y * this.dimensions.frame.width) + x;
+        cell = this.tty_grid.cells[index];
+        if (cell === undefined) {
+          this.frame.colours.push(0)
+          this.frame.colours.push(0)
+          this.frame.colours.push(0)
+          this.frame.text.push("")
+        } else {
+          cell.fg_colour.map((c) => this.frame.colours.push(c));
+          this.frame.text.push(cell.rune);
+        }
+      }
+    }
+  }
+
+  _serialiseRawText() {
+    let raw_text = "";
+    this._previous_cell_href = '';
+    this._is_inside_anchor = false;
+    const top = this.dimensions.frame.sub.top / 2;
+    const left = this.dimensions.frame.sub.left;
+    const bottom = top + (this.dimensions.frame.sub.height / 2);
+    const right = left + this.dimensions.frame.sub.width;
+    for (let y = top; y < bottom; y++) {
+      for (let x = left; x < right; x++) {
+        raw_text += this._addCell(x, y, right);
+      }
+      raw_text += "\n";
+    }
+    const head = `<html><title>${document.title}</title><body><pre>`
+    const foot = `</body></pre></html>`
+    return head + raw_text + foot;
+  }
+
+  // TODO: Ultimately we're going to need to know exactly which parts of the input
+  //       box are obscured. This is partly possible using the element's computed
+  //       styles, however this isn't comprehensive - think partially obscuring.
+  //       So the best solution is to use the same trick as we do for normal text,
+  //       except that we can't fill the input box with text, however we can
+  //       temporarily change the background to a contrasting colour.
+  _getAllInputBoxes() {
+    let dom_rect, styles, font_rgb;
+    let parsed_input_boxes = {};
+    let raw_input_boxes = document.querySelectorAll(
+      'input, ' +
+      'textarea, ' +
+      '[role="textbox"]'
+    );
+    raw_input_boxes.forEach((i) => {
+      let type;
+      this._ensureBrowshID(i);
+      dom_rect = this._convertDOMRectToAbsoluteCoords(i.getBoundingClientRect());
+      const width = utils.snap(dom_rect.width * this.dimensions.scale_factor.width);
+      const height = utils.snap(dom_rect.height * this.dimensions.scale_factor.height);
+      if (width == 0 || height == 0) { return }
+      if (i.getAttribute('role') == 'textbox') {
+        type = 'textbox';
+      } else {
+        type = i.getAttribute('type');
+      }
+      styles = window.getComputedStyle(i);
+      font_rgb = styles['color'].replace(/[^\d,]/g, '').split(',').map((i) => parseInt(i));
+      if (this._isUnwantedInboxBox(i, styles)) { return }
+      parsed_input_boxes[i.getAttribute('data-browsh-id')] = {
+        id: i.getAttribute('data-browsh-id'),
+        x: utils.snap(dom_rect.left * this.dimensions.scale_factor.width),
+        y: utils.snap(dom_rect.top * this.dimensions.scale_factor.height),
+        width: width,
+        height: height,
+        tag_name: i.nodeName,
+        type: type,
+        colour: [font_rgb[0], font_rgb[1], font_rgb[2]]
+      };
+    });
+    return parsed_input_boxes;
+  }
+
+  _ensureBrowshID(element) {
+    if (element.getAttribute('data-browsh-id') === null) {
+      element.setAttribute('data-browsh-id', utils.uuidv4());
+    }
+  }
+
+  _isUnwantedInboxBox(input_box, styles) {
+    if (styles.display === 'none' || styles.visibility === 'hidden') { return true }
+    if (input_box.getAttribute('aria-hidden') == 'true') { return true }
+    return false;
+  }
+
+  _sendRawText() {
+    let payload = {
+      body: this._serialiseRawText()
+    }
+    this.sendMessage(`/raw_text,${JSON.stringify(payload)}`);
+  }
+
+  _sendFrame() {
+    this._serialiseFrame();
+    if (this.frame.text.length > 0) {
+      this.sendMessage(`/frame_text,${JSON.stringify(this.frame)}`);
+    } else {
+      this.log("Not sending empty text frame");
+    }
+  }
+
+  _addCell(x, y, right) {
+    let text = "";
+    const index = (y * this.dimensions.frame.width) + x;
+    this._cell_for_raw_text = this.tty_grid.cells[index];
+    if (this._raw_mode_type === 'raw_text_html') {
+      this._is_line_end = (x === right - 1);
+      text += this._addCellAsHTML();
+    } else {
+      text += this._addCellAsPlainText();
+    }
+    return text;
+  }
+
+  _addCellAsHTML() {
+    this._HTML = '';
+    if (!this._cell_for_raw_text) {
+      this._addHTMLForNonExistentCell();
+      return this._HTML;
+    }
+    this._current_cell_href = this._cell_for_raw_text.parent_element.href;
+    this._is_HREF_changed = this._current_cell_href !== this._previous_cell_href
+    this._handleCellOutsideAnchor();
+    this._handleCellInsideAnchor();
+    this._HTML += this._cell_for_raw_text.rune;
+    if (this._will_be_inside_anchor !== undefined) {
+      this._is_inside_anchor = this._will_be_inside_anchor;
+    }
+    this._previous_cell_href = this._current_cell_href;
+    return this._HTML;
+  }
+
+  _addHTMLForNonExistentCell() {
+    if (this._is_inside_anchor) {
+      this._previous_cell_href = undefined;
+      this._closeAnchorTag();
+    }
+    this._HTML += " ";
+  }
+
+  _handleCellOutsideAnchor() {
+    if (this._is_inside_anchor) { return }
+    if (this._current_cell_href || this._is_HREF_changed) {
+      this._openAnchorTag();
+    }
+  }
+
+  _handleCellInsideAnchor() {
+    if (!this._is_inside_anchor) { return }
+    if (this._is_HREF_changed || !this._current_cell_href || this._is_line_end) {
+      this._closeAnchorTag();
+      if (this._current_cell_href) {
+        this._openAnchorTag();
+      }
+    }
+  }
+
+  _openAnchorTag() {
+    this._will_be_inside_anchor = true;
+    this._HTML += `<a href="/${this._current_cell_href}">`;
+  }
+
+  _closeAnchorTag() {
+    this._will_be_inside_anchor = false;
+    this._HTML += `</a>`;
+  }
+
+  _addCellAsPlainText() {
+    if (this._cell_for_raw_text === undefined) { return " " }
+    return this._cell_for_raw_text.rune;
+  }
+
+  _setupFrameMeta() {
+    this.frame = {
+      meta: this.dimensions.getFrameMeta(),
+      text: [],
+      colours: []
+    };
+    this.frame.meta.id = parseInt(this.channel.name)
+  }
+
+  _serialiseInputBoxes() {
+    this.frame.input_boxes = this._getAllInputBoxes();
+  }
+}

webext/src/dom/text_builder.js

@@ -2,12 +2,13 @@ import _ from 'lodash';
 
 import utils from 'utils';
 import CommonMixin from 'dom/common_mixin';
+import SerialiseMixin from 'dom/serialise_mixin';
 import TTYCell from 'dom/tty_cell';
 import TTYGrid from 'dom/tty_grid';
 
 // Convert the text on the page into a snapped 2-dimensional grid to be displayed directly
 // in the terminal.
-export default class extends utils.mixins(CommonMixin) {
+export default class extends utils.mixins(CommonMixin, SerialiseMixin) {
   constructor(channel, dimensions, graphics_builder) {
     super();
     this.channel = channel;
@@ -26,7 +27,8 @@ export default class extends utils.mixins(CommonMixin) {
     this._sendFrame();
   }
 
-  sendRawText() {
+  sendRawText(type) {
+    this._raw_mode_type = type;
     // TODO:
     //   The presence of the `getScreenshotWithText()` and `setTimeout()` calls are a hack
     //   that I am unable to understand the reasoning for - unfortunately they came about
@@ -354,137 +356,6 @@ export default class extends utils.mixins(CommonMixin) {
     }
   }
 
-  // TODO: Ultimately we're going to need to know exactly which parts of the input
-  //       box are obscured. This is partly possible using the element's computed
-  //       styles, however this isn't comprehensive - think partially obscuring.
-  //       So the best solution is to use the same trick as we do for normal text,
-  //       except that we can't fill the input box with text, however we can
-  //       temporarily change the background to a contrasting colour.
-  _getAllInputBoxes() {
-    let dom_rect, styles, font_rgb;
-    let parsed_input_boxes = {};
-    let raw_input_boxes = document.querySelectorAll(
-      'input, ' +
-      'textarea, ' +
-      '[role="textbox"]'
-    );
-    raw_input_boxes.forEach((i) => {
-      let type;
-      this._ensureBrowshID(i);
-      dom_rect = this._convertDOMRectToAbsoluteCoords(i.getBoundingClientRect());
-      const width = utils.snap(dom_rect.width * this.dimensions.scale_factor.width);
-      const height = utils.snap(dom_rect.height * this.dimensions.scale_factor.height);
-      if (width == 0 || height == 0) { return }
-      if (i.getAttribute('role') == 'textbox') {
-        type = 'textbox';
-      } else {
-        type = i.getAttribute('type');
-      }
-      styles = window.getComputedStyle(i);
-      font_rgb = styles['color'].replace(/[^\d,]/g, '').split(',').map((i) => parseInt(i));
-      if (this._isUnwantedInboxBox(i, styles)) { return }
-      parsed_input_boxes[i.getAttribute('data-browsh-id')] = {
-        id: i.getAttribute('data-browsh-id'),
-        x: utils.snap(dom_rect.left * this.dimensions.scale_factor.width),
-        y: utils.snap(dom_rect.top * this.dimensions.scale_factor.height),
-        width: width,
-        height: height,
-        tag_name: i.nodeName,
-        type: type,
-        colour: [font_rgb[0], font_rgb[1], font_rgb[2]]
-      };
-    });
-    return parsed_input_boxes;
-  }
-
-  _ensureBrowshID(element) {
-    if (element.getAttribute('data-browsh-id') === null) {
-      element.setAttribute('data-browsh-id', utils.uuidv4());
-    }
-  }
-
-  _isUnwantedInboxBox(input_box, styles) {
-    if (styles.display === 'none' || styles.visibility === 'hidden') { return true }
-    if (input_box.getAttribute('aria-hidden') == 'true') { return true }
-    return false;
-  }
-
-  _sendRawText() {
-    let payload = {
-      body: this._serialiseRawText()
-    }
-    this.sendMessage(`/raw_text,${JSON.stringify(payload)}`);
-  }
-
-  _sendFrame() {
-    this._serialiseFrame();
-    if (this.frame.text.length > 0) {
-      this.sendMessage(`/frame_text,${JSON.stringify(this.frame)}`);
-    } else {
-      this.log("Not sending empty text frame");
-    }
-  }
-
-  __serialiseFrame() {
-    let cell, index;
-    const top = this.dimensions.frame.sub.top / 2;
-    const left = this.dimensions.frame.sub.left;
-    const bottom = top + (this.dimensions.frame.sub.height / 2);
-    const right = left + this.dimensions.frame.sub.width;
-    this._setupFrameMeta();
-    this._serialiseInputBoxes();
-    for (let y = top; y < bottom; y++) {
-      for (let x = left; x < right; x++) {
-        index = (y * this.dimensions.frame.width) + x;
-        cell = this.tty_grid.cells[index];
-        if (cell === undefined) {
-          this.frame.colours.push(0)
-          this.frame.colours.push(0)
-          this.frame.colours.push(0)
-          this.frame.text.push("")
-        } else {
-          cell.fg_colour.map((c) => this.frame.colours.push(c));
-          this.frame.text.push(cell.rune);
-        }
-      }
-    }
-  }
-
-  _serialiseRawText() {
-    let cell, index;
-    let raw_text = "";
-    const top = this.dimensions.frame.sub.top / 2;
-    const left = this.dimensions.frame.sub.left;
-    const bottom = top + (this.dimensions.frame.sub.height / 2);
-    const right = left + this.dimensions.frame.sub.width;
-    for (let y = top; y < bottom; y++) {
-      for (let x = left; x < right; x++) {
-        index = (y * this.dimensions.frame.width) + x;
-        cell = this.tty_grid.cells[index];
-        if (cell) {
-          raw_text += cell.rune;
-        } else {
-          raw_text += " ";
-        }
-      }
-      raw_text += "\n";
-    }
-    return raw_text;
-  }
-
-  _setupFrameMeta() {
-    this.frame = {
-      meta: this.dimensions.getFrameMeta(),
-      text: [],
-      colours: []
-    };
-    this.frame.meta.id = parseInt(this.channel.name)
-  }
-
-  _serialiseInputBoxes() {
-    this.frame.input_boxes = this._getAllInputBoxes();
-  }
-
   // Purely for debugging.
   //
   // Draws a red border around all the DOMClientRect nodes.