beenull
/
browsh
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compose frames of whole DOM, for faster scrolling 

Instead of mirroring the browser's viewport, as if we had a camera over
the browser, the entire DOM is now sent in the frame. This means that
the CLI itself can scroll without having to wait for updates from the
webextension screenshotter.
This commit is contained in:
Thomas Buckley-Houston 2018-04-18 21:11:04 +08:00
parent 5b6cc89770
commit c0f5fcca69
27 changed files with 1237 additions and 817 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
interfacer/src/browsh/browsh.go
View File

@ -4,6 +4,7 @@ import (
"encoding/base64"
"flag"
"fmt"
"strconv"
"io/ioutil"
"net"
"net/http"
@ -42,6 +43,11 @@ var (
ffCommandCount = 0
isConnectedToWebExtension = false
screen tcell.Screen
frame []string
uiHeight = 2
frameWidth int
frameHeight int
State map[string]string
defaultFFPrefs = map[string]string{
"browser.startup.homepage": "'https://www.google.com'",
"startup.homepage_welcome_url": "'https://www.google.com'",
@ -207,6 +213,22 @@ func Start(injectedScreen tcell.Screen) {
Log("Exiting at end of main()")
}
func toInt(char string) int {
i, err := strconv.ParseInt(char, 10, 16)
if err != nil {
Shutdown(err)
}
return int(i)
}
func toInt32(char string) int32 {
i, err := strconv.ParseInt(char, 10, 32)
if err != nil {
Shutdown(err)
}
return int32(i)
}
// TtyStart ... Main entrypoint.
func TtyStart() {
// Hack to force true colours

34
interfacer/src/browsh/comms.go
View File

@ -39,8 +39,16 @@ func handleWebextensionCommand(message []byte) {
command := parts[0]
switch command {
case "/frame":
frame := parseJSONframe(strings.Join(parts[1:], ","))
renderFrame(frame)
frame = parseJSONFrame(strings.Join(parts[1:], ","))
renderUI()
renderFrame()
case "/state":
oldState := map[string]string{}
for k,v := range State{
oldState[k] = v
}
parseJSONState(strings.Join(parts[1:], ","))
handleStateChange(oldState)
case "/screenshot":
saveScreenshot(parts[1])
default:
@ -51,7 +59,7 @@ func handleWebextensionCommand(message []byte) {
// Frames received from the webextension are 1 dimensional arrays of strings.
// They are made up of a repeating pattern of 7 items:
// ["FG RED", "FG GREEN", "FG BLUE", "BG RED", "BG GREEN", "BG BLUE", "CHARACTER" ...]
func parseJSONframe(jsonString string) []string {
func parseJSONFrame(jsonString string) []string {
var frame []string
jsonBytes := []byte(jsonString)
if err := json.Unmarshal(jsonBytes, &frame); err != nil {
@ -60,6 +68,26 @@ func parseJSONframe(jsonString string) []string {
return frame
}
func parseJSONState(jsonString string) {
jsonBytes := []byte(jsonString)
if err := json.Unmarshal(jsonBytes, &State); err != nil {
Shutdown(err)
}
}
func handleStateChange(oldState map[string]string) {
if (State["page_state"] != oldState["page_state"]) {
Log("State change: page_state=" + State["page_state"])
if (State["page_state"] == "page_init") {
yScroll = 0
}
}
if (State["frame_width"] != "" && State["frame_height"] != "") {
frameWidth = toInt(State["frame_width"])
frameHeight = toInt(State["frame_height"])
}
}
// When the socket reader attempts to read from a closed websocket it quickly and
// simply closes its associated Go routine. However the socket writer won't
// automatically notice until it actually needs to send something. So we force that

159
interfacer/src/browsh/tty.go
View File

@ -3,13 +3,17 @@ package browsh
import (
"fmt"
"os"
"strconv"
"encoding/json"
"github.com/gdamore/tcell"
"github.com/go-errors/errors"
)
var (
xScroll = 0
yScroll = 0
)
func setupTcell() {
var err error
if err = screen.Init(); err != nil {
@ -32,12 +36,7 @@ func readStdin() {
ev := screen.PollEvent()
switch ev := ev.(type) {
case *tcell.EventKey:
if ev.Key() == tcell.KeyCtrlQ {
if !*isUseExistingFirefox {
quitFirefox()
}
Shutdown(errors.New("normal"))
}
handleUserKeyPress(ev)
eventMap := map[string]interface{}{
"key": int(ev.Key()),
"char": string(ev.Rune()),
@ -53,8 +52,8 @@ func readStdin() {
button := ev.Buttons()
eventMap := map[string]interface{}{
"button": int(button),
"mouse_x": int(x),
"mouse_y": int(y),
"mouse_x": int(x + xScroll),
"mouse_y": int(y - uiHeight + yScroll),
"modifiers": int(ev.Modifiers()),
}
marshalled, _ := json.Marshal(eventMap)
@ -63,6 +62,44 @@ func readStdin() {
}
}
func handleUserKeyPress(ev *tcell.EventKey) {
if ev.Key() == tcell.KeyCtrlQ {
if !*isUseExistingFirefox {
quitFirefox()
}
Shutdown(errors.New("normal"))
}
handleScrolling(ev)
}
func handleScrolling(ev *tcell.EventKey) {
yScrollOriginal := yScroll
_, height := screen.Size()
height -= uiHeight
if ev.Key() == tcell.KeyUp {
yScroll -= 2
}
if ev.Key() == tcell.KeyDown {
yScroll += 2
}
if ev.Key() == tcell.KeyPgUp {
yScroll -= height
}
if ev.Key() == tcell.KeyPgDn {
yScroll += height
}
limitScroll(height)
if (yScroll != yScrollOriginal) {
renderFrame()
}
}
func limitScroll(height int) {
maxYScroll := (frameHeight / 2) - height
if (yScroll > maxYScroll) { yScroll = maxYScroll }
if (yScroll < 0) { yScroll = 0 }
}
// Write a simple text string to the screen. Not for use in the browser frames
// themselves. If you want anything to appear in the browser that must be done
// through the webextension.
@ -77,26 +114,24 @@ func writeString(x, y int, str string) {
screen.Sync()
}
// Tcell uses a buffer to collect screen updates on, it only actually sends
// ANSI rendering commands to the terminal when we tell it to. And even then it
// will try to minimise rendering commands by only rendering parts of the terminal
// that have changed.
func renderFrame(frame []string) {
func renderAll() {
renderUI()
renderFrame()
}
// Render the tabs and URL bar
// TODO: Temporary function, UI rendering should all be moved into this CLI app
func renderUI() {
var styling = tcell.StyleDefault
var character string
var runeChars []rune
width, height := screen.Size()
if (width * height * 7 != len(frame)) {
Log("Not rendering frame: current frame is not the same size as the screen")
Log(fmt.Sprintf("screen: %d, frame: %d", width * height * 7, len(frame)))
return
}
width, _ := screen.Size()
index := 0
for y := 0; y < height; y++ {
for y := 0; y < uiHeight ; y++ {
for x := 0; x < width; x++ {
styling = styling.Foreground(getRGBColor(frame, index))
styling = styling.Foreground(getRGBColor(index))
index += 3
styling = styling.Background(getRGBColor(frame, index))
styling = styling.Background(getRGBColor(index))
index += 3
character = frame[index]
runeChars = []rune(character)
@ -110,21 +145,79 @@ func renderFrame(frame []string) {
screen.Show()
}
// Tcell uses a buffer to collect screen updates on, it only actually sends
// ANSI rendering commands to the terminal when we tell it to. And even then it
// will try to minimise rendering commands by only rendering parts of the terminal
// that have changed.
func renderFrame() {
var styling = tcell.StyleDefault
var character string
var runeChars []rune
width, height := screen.Size()
uiSize := uiHeight * width * 7
if (len(frame) == uiSize) {
Log("Not rendering zero-size frame data")
return
}
if (frameWidth == 0 || frameHeight == 0) {
Log("Not rendering frame with a zero dimension")
return
}
index := 0
for y := 0; y < height - uiHeight; y++ {
for x := 0; x < width; x++ {
index = ((y + yScroll) * frameWidth * 7) + ((x + xScroll) * 7)
index += uiSize
if (!checkCell(index, x + xScroll, y + yScroll)) { return }
styling = styling.Foreground(getRGBColor(index))
index += 3
styling = styling.Background(getRGBColor(index))
index += 3
character = frame[index]
runeChars = []rune(character)
index++
if (character == "WIDE") {
continue
}
screen.SetCell(x, y + uiHeight, styling, runeChars[0])
}
}
overlayPageStatusMessage(height)
screen.Show()
}
func overlayPageStatusMessage(height int) {
message := State["page_status_message"]
x := 0
fg := tcell.NewHexColor(int32(0xffffff))
bg := tcell.NewHexColor(int32(0x000000))
style := tcell.StyleDefault
style.Foreground(fg)
style.Foreground(bg)
for _, c := range message {
screen.SetCell(x, height - 1, style, c)
x++
}
}
func checkCell(index, x, y int) bool {
for i := 0; i < 7; i++ {
if (index + i >= len(frame) || frame[index + i] == "") {
message := fmt.Sprintf("Blank frame data (size: %d) at %dx%d, index:%d/%d", len(frame), x, y, index, i)
Log(message)
Log(fmt.Sprintf("%d", yScroll))
return false;
}
}
return true;
}
// Given a raw frame from the webextension, find the RGB colour at a given
// 1 dimensional index.
func getRGBColor(frame []string, index int) tcell.Color {
func getRGBColor(index int) tcell.Color {
rgb := frame[index:index + 3]
return tcell.NewRGBColor(
toInt32(rgb[0]),
toInt32(rgb[1]),
toInt32(rgb[2]))
}
// Convert a string representation of an integer to an integer
func toInt32(char string) int32 {
i, err := strconv.ParseInt(char, 10, 32)
if err != nil {
Shutdown(err)
}
return int32(i)
}

18
interfacer/test/browsh_test.go
View File

@ -36,23 +36,31 @@ var _ = Describe("Showing a basic webpage", func() {
})
It("should navigate to a new page by clicking a link", func() {
simScreen.InjectMouse(12, 21, tcell.Button1, tcell.ModNone)
Expect("Another▄page").To(BeInFrameAt(12, 19))
simScreen.InjectMouse(12, 19, tcell.Button1, tcell.ModNone)
Expect("Another").To(BeInFrameAt(0, 0))
})
It("should scroll the page by one line", func() {
SpecialKey(tcell.KeyDown)
Expect("meal,▄originating▄in▄").To(BeInFrameAt(12, 12))
Expect("meal,▄originating▄in▄").To(BeInFrameAt(12, 11))
})
It("should scroll the page by one page", func() {
SpecialKey(tcell.KeyPgDn)
Expect("continuing▄with▄a▄variety▄of▄fish").To(BeInFrameAt(12, 10))
Expect("continuing▄with▄a▄variety▄of▄fish").To(BeInFrameAt(12, 12))
})
})
})
Describe("Rendering", func() {
It("should reset page scroll to zero on page load", func() {
SpecialKey(tcell.KeyPgDn)
Expect("continuing▄with▄a▄variety▄of▄fish").To(BeInFrameAt(12, 12))
GotoURL(testSiteURL + "/smorgasbord/another.html")
Expect("Another▄webpage").To(BeInFrameAt(1, 3))
})
It("should render dynamic content", func() {
Expect([3]int32{0, 255, 255}).To(Equal(GetFgColour(39, 3)))
waitForNextFrame()
@ -68,8 +76,8 @@ var _ = Describe("Showing a basic webpage", func() {
Describe("Text positioning", func() {
It("should position the left/right-aligned coloumns", func() {
Expect("Smörgåsbord▄(Swedish:").To(BeInFrameAt(12, 11))
Expect("The▄Swedish▄word").To(BeInFrameAt(42, 11))
Expect("Smörgåsbord▄(Swedish:").To(BeInFrameAt(12, 10))
Expect("The▄Swedish▄word").To(BeInFrameAt(42, 10))
})
})
})

16
interfacer/test/setup.go
View File

@ -85,13 +85,25 @@ func WaitForText(text string, x, y int) {
panic("Waiting for '" + text + "' to appear but it didn't")
}
// WaitForPageLoad waits for the page to load
func WaitForPageLoad() {
start := time.Now()
for time.Since(start) < perTestTimeout {
if browsh.State["page_state"] == "parsing_complete" {
time.Sleep(100 * time.Millisecond)
return
}
time.Sleep(100 * time.Millisecond)
}
panic("Page didn't load within timeout")
}
// GotoURL sends the browsh browser to the specified URL
func GotoURL(url string) {
SpecialKey(tcell.KeyCtrlL)
Keyboard(url)
SpecialKey(tcell.KeyEnter)
WaitForText("Loading", 0, 24)
WaitForText("▄▄▄▄▄▄▄", 0, 24)
WaitForPageLoad()
// TODO: Looking for the URL isn't optimal because it could be the same URL
// as the previous test.
gomega.Expect(url).To(BeInFrameAt(9, 1))

3
webext/.eslintrc
View File

@ -8,7 +8,8 @@
},
"globals": {
"DEVELOPMENT": true,
"PRODUCTION": true
"PRODUCTION": true,
"TEST": true
},
"parser": "babel-eslint",
"parserOptions": {

8
webext/src/background/common_mixin.js
View File

@ -1,3 +1,4 @@
import _ from 'lodash';
import stripAnsi from 'strip-ansi';
// Here we keep the public functions used to mediate communications between
@ -11,9 +12,14 @@ export default (MixinBase) => class extends MixinBase {
this.terminal.send(message);
}
sendState() {
const state = _.mapValues(this.state, (v) => { return v.toString() });
this.sendToTerminal(`/state,${JSON.stringify(state)}`);
}
log(...messages) {
if (messages.length === 1) {
messages = messages[0];
messages = messages[0].toString();
messages = stripAnsi(messages);
messages = messages.replace(/\u001b\[/g, 'ESC');
}

16
webext/src/background/manager.js
View File

@ -9,12 +9,22 @@ import TabCommandsMixin from 'background/tab_commands_mixin';
export default class extends utils.mixins(CommonMixin, TTYCommandsMixin, TabCommandsMixin) {
constructor() {
super();
// All state that needs to be synced with the TTY
this.state = {}
// Dimensions from the tab
this.dimensions = {
dom: {},
char: {},
frame: {}
}
// Keep track of connections to active tabs
this.tabs = {};
// The ID of the tab currently opened in the browser
this.active_tab_id = null;
// Keep track of automatic reloads to problematic tabs
this._tab_reloads = [];
// When the real GUI browser first launches it's sized to the same size as the desktop
this._is_initial_window_pending = true;
this._connectToTerminal();
}
@ -197,10 +207,10 @@ export default class extends utils.mixins(CommonMixin, TTYCommandsMixin, TabComm
// efficient way of triggering this initial window resize, than just waiting for the data
// on every frame tick.
_initialWindowResize() {
if (!this._is_intial_window_pending) return;
if (!this._is_initial_window_pending) return;
if(this.char_width && this.char_height && this.tty_width && this.tty_height) {
this.resizeBrowserWindow();
this._is_intial_window_pending = false;
this._is_initial_window_pending = false;
}
}
@ -214,7 +224,7 @@ export default class extends utils.mixins(CommonMixin, TTYCommandsMixin, TabComm
this.log("Not sending frame to TTY without TTY size")
return;
}
if (this._is_intial_window_pending) this._initialWindowResize();
if (this._is_initial_window_pending) this._initialWindowResize();
if (!this.tabs.hasOwnProperty(this.active_tab_id)) {
this.log("No active tab, so not requesting a frame");
return;

59
webext/src/background/tab_commands_mixin.js
View File

@ -6,6 +6,7 @@ import utils from 'utils';
// the current character dimensions .
export default (MixinBase) => class extends MixinBase {
handleTabMessage(message) {
let incoming;
const parts = message.split(',');
const command = parts[0];
switch (command) {
@ -17,13 +18,11 @@ export default (MixinBase) => class extends MixinBase {
case '/tab_info':
this.currentTab().info = JSON.parse(utils.rebuildArgsToSingleArg(parts));
break;
case '/char_size':
this.char_width = parts[1];
this.char_height = parts[2]
if(this.tty_width && this.tty_height) this.resizeBrowserWindow();
break;
case '/request_tty_size':
this.sendTTYSizeToBrowser();
case '/dimensions':
incoming = JSON.parse(message.slice(12));
this._mightResizeWindow(incoming);
this.dimensions = incoming;
this._sendFrameSize();
break;
case '/status':
if (this._current_frame) {
@ -39,55 +38,53 @@ export default (MixinBase) => class extends MixinBase {
}
}
sendTTYSizeToBrowser() {
this.sendToCurrentTab(`/tty_size,${this.tty_width},${this.tty_height}`);
_mightResizeWindow(incoming) {
if (this.dimensions.char.width != incoming.char.width ||
this.dimensions.char.height != incoming.char.height) {
this.dimensions = incoming;
this.resizeBrowserWindow();
}
}
_sendFrameSize() {
this.state['frame_width'] = this.dimensions.frame.width;
this.state['frame_height'] = this.dimensions.frame.height;
this.sendState();
}
_sendCurrentFrame() {
this.sendState();
// TODO: I struggled with unmarshalling a mixed array in Golang so I'm crudely
// just casting evertything to a string for now.
// just casting everything to a string for now.
this._current_frame = this._current_frame.map((i) => i.toString());
this.sendToTerminal(`/frame,${JSON.stringify(this._current_frame)}`);
}
updateStatus(status, message = '') {
if (typeof this._current_frame === 'undefined') return;
let status_message;
switch (status) {
case 'page_init':
this._page_status = `Loading ${this.currentTab().info.url}`;
status_message = `Loading ${this.currentTab().info.url}`;
break;
case 'parsing_complete':
this._page_status = '';
status_message = '';
break;
case 'window_unload':
this._page_status = 'Loading...';
status_message = 'Loading...';
break;
default:
if (message != '') status = message;
this._page_status = status;
if (message != '') status_message = message;
}
this._applyStatus();
this.state['page_state'] = status;
this.state['page_status_message'] = status_message;
this.sendState();
}
_applyUI() {
const tabs = this._buildTTYRow(this._buildTabs());
const urlBar = this._buildURLBar();
this._current_frame = tabs.concat(urlBar).concat(this._current_frame);
this._applyStatus();
}
_applyStatus() {
if (typeof this._page_status === 'undefined') return;
let cell;
const cell_item_count = 7
const bottom_line = this.tty_height - 1;
const start = bottom_line * this.tty_width * cell_item_count;
for (let i = 0; i < this.tty_width; i++) {
if (this._page_status[i] !== undefined) {
cell = utils.ttyPlainCell(this._page_status[i]);
this._current_frame.splice(start + (i * 7), 7, ...cell)
}
}
}
_buildTabs() {

12
webext/src/background/tty_commands_mixin.js
View File

@ -16,9 +16,7 @@ export default (MixinBase) => class extends MixinBase {
case '/tty_size':
this.tty_width = parts[1];
this.tty_height = parts[2];
if (this.char_width && this.char_height){
this.resizeBrowserWindow();
}
this.resizeBrowserWindow();
break;
case '/stdin':
if (!this._handleUICommand(parts)) {
@ -115,20 +113,20 @@ export default (MixinBase) => class extends MixinBase {
}
resizeBrowserWindow() {
if (!this.tty_width || !this.char_width || !this.tty_height || !this.char_height) {
if (!this.tty_width || !this.dimensions.char.width || !this.tty_height || !this.dimensions.char.height) {
this.log(
'Not resizing browser window without all of the TTY and character dimensions'
);
return;
}
// Does this include scrollbars???
const window_width = parseInt(Math.round(this.tty_width * this.char_width));
// Leave room for tabs and URL bar. TODO: globally refactor TTY DOM height
const window_width = parseInt(Math.round(this.tty_width * this.dimensions.char.width));
// Leave room for tabs and URL bar
const tty_dom_height = this.tty_height - 2;
// I don't know why we have to add 4 more lines to the window height?? But without
// it text doesn't fill the bottom of the TTY.
const window_height = parseInt(Math.round(
(tty_dom_height + 4) * this.char_height
(tty_dom_height + 4) * this.dimensions.char.height
));
const current_window = browser.windows.getCurrent();
current_window.then(

12
webext/src/dom/common_mixin.js
View File

@ -1,10 +1,12 @@
export default (MixinBase) => class extends MixinBase {
sendMessage(message) {
this.channel.postMessage(message);
constructor() {
super();
this._is_first_frame_finished = false;
}
snap(number) {
return parseInt(Math.round(number));
sendMessage(message) {
if (this.channel == undefined) { return }
this.channel.postMessage(message);
}
log(...messages) {
@ -21,7 +23,7 @@ export default (MixinBase) => class extends MixinBase {
// If you're logging large objects and using a high-ish FPS (<1000ms) then you might
// crash the browser. So use this function instead.
firstFrameLog(...logs) {
if (this._is_first_frame_finished) return;
//if (this._is_first_frame_finished) return;
if (DEVELOPMENT) {
this.log(logs);
}

141
webext/src/dom/dimensions.js Normal file
View File

@ -0,0 +1,141 @@
import utils from 'utils';
import CommonMixin from 'dom/common_mixin';
// All the various dimensions, sizes, scales, etc
export default class extends utils.mixins(CommonMixin) {
constructor() {
super()
// ID for element we place in the DOM to measure the size of a single monospace
// character.
this._measuring_box_id = 'browsh_em_measuring_box';
// TODO: WTF is this magic number?
if (TEST) {
this._char_height_magic_number = 2
} else {
this._char_height_magic_number = 4
}
this.dom = {};
if (document.body) {
this.update();
}
}
update() {
this._calculateCharacterDimensions();
this._updateDOMDimensions();
this._calculateScaleFactor();
this._updateFrameDimensions();
this._notifyBackground();
}
// This is critical in order for the terminal to match the browser as closely as possible.
// Ideally we want the browser's window size to be exactly multiples of the terminal's
// dimensions. So if the terminal is 80x40 and the font-size is 12px (12x6 pixels), then
// the window should be 480x480. Also knowing the precise font-size helps the text builder
// map un-snapped text to the best grid cells - grid cells that represent the terminal's
// character positions.
//
// The reason that we can't just do some basic maths on the CSS `font-size` value we enforce
// is that there are various factors that can skew the actual font dimensions on the page.
// For instance, different browser families and even different versions of the same browser
// may have subtle differences in how they render text. Furthermore we can actually get
// floating point accuracy if we use `Element.getBoundingClientRect()` which further helps
// as calculations are compounded during our rendering processes.
_calculateCharacterDimensions() {
const element = this._getOrCreateMeasuringBox();
const dom_rect = element.getBoundingClientRect();
this.char = {
width: dom_rect.width,
height: dom_rect.height + this._char_height_magic_number
}
}
// Back when printing was done by physical stamps, it was convention to measure the
// font-size using the letter 'M', thus where we get the unit 'em' from. Not that it
// should not make any difference to us, but it's nice to keep a tradition.
_getOrCreateMeasuringBox() {
let measuring_box = this.findMeasuringBox();
if (measuring_box) return measuring_box;
measuring_box = document.createElement('span');
measuring_box.id = this._measuring_box_id;
measuring_box.style.visibility = 'hidden';
var M = document.createTextNode('M');
measuring_box.appendChild(M);
document.body.appendChild(measuring_box);
return measuring_box;
}
findMeasuringBox() {
return document.getElementById(this._measuring_box_id);
}
_updateDOMDimensions() {
const [new_width, new_height] = this._calculateDOMDimensions();
const is_new = this.dom.width != new_width || this.dom.height != new_height
this.dom = {
// Even though it is the TTY's responsibility to scroll the DOM, the browser still
// needs to do scrolling because various events can be triggered by it - think of
// lazy image loading.
x_scroll: window.scrollX,
y_scroll: window.scrollY,
width: new_width,
height: new_height,
is_new: is_new
}
}
// For discussion on various methods to get total scrollable DOM dimensions, see:
// https://stackoverflow.com/a/44077777/575773
_calculateDOMDimensions() {
let width = document.documentElement.scrollWidth;
if (window.innerWidth > width) width = window.innerWidth;
let height = document.documentElement.scrollHeight;
if (window.innerHeight > height) height = window.innerHeight;
return [width, height]
}
// A frame represents the entire DOM page. Its height usually extends below the window's
// bottom and occasionally extends beyond the sides too.
//
// Note that it treats the height of a single TTY cell as containing 2 pixels. Therefore
// a TTY of 4x4 will have frame dimensions of 4x8.
_updateFrameDimensions() {
this.frame = {
width: utils.snap(this.dom.width * this.scale_factor.width),
height: utils.snap(this.dom.height * this.scale_factor.height)
}
}
// The scale factor is the ratio of the TTY's representation of the DOM to the browser's
// representation of the DOM. The idea is that the TTY just represents a very low
// resolution version of the browser - though note that the TTY has the significant
// benefit of being able to display native fonts (possibly even retina-like high DPI
// fonts). So Browsh's enforced CSS rules reorient the browser page to render all text
// at the same monospaced sized - in this sense, theoretically, the TTY and the browser
// should essentially be facsimilies of each other. However of course the TTY is limited
// by its cell size in how it renders "pixels", namely pseudo pixels using the UTF8
// block trick.
//
// All of which is to say that the fundamental relationship between the browser's dimensions
// and the TTY's dimensions is represented by a TTY cell - that which displays a single
// character. So if we know how many characters fit into the DOM, then we know how many
// "pixels" the TTY should have.
_calculateScaleFactor() {
this.scale_factor = {
width: 1 / this.char.width,
// Recall that 2 UTF8 half-black "pixels" can fit into a single TTY cell
height: 2 / this.char.height
}
}
_notifyBackground() {
const dimensions = {
dom: this.dom,
frame: this.frame,
char: this.char
}
this.sendMessage(`/dimensions,${JSON.stringify(dimensions)}`)
}
}

143
webext/src/dom/document_builder.js
View File

@ -1,143 +0,0 @@
import charWidthInTTY from 'string-width';
import utils from 'utils';
import CommonMixin from 'dom/common_mixin';
import GraphicsBuilderMixin from 'dom/graphics_builder_mixin';
import TextBuilderMixin from 'dom/text_builder_mixin';
// Takes the graphics and text from the current DOM, combines them, then
// sends it to the background process where the rest of the UI, like tabs,
// address bar, etc will be added.
export default class extends utils.mixins(CommonMixin, GraphicsBuilderMixin, TextBuilderMixin) {
constructor(channel) {
super();
this.channel = channel;
this.is_graphics_mode = true;
}
makeFrame() {
this._setupDimensions();
this._compileFrame();
this._buildFrame();
}
_compileFrame() {
this.getScreenshotWithText();
this.getScreenshotWithoutText();
this.getScaledScreenshot();
this.buildFormattedText();
}
_buildFrame() {
this.logPerformance(() => {
this.__buildFrame();
}, 'build frame');
}
__buildFrame() {
this.frame = [];
this._bg_row = [];
this._fg_row = [];
for (let y = 0; y < this.frame_height; y++) {
for (let x = 0; x < this.frame_width; x++) {
this._buildPixel(x, y);
}
}
}
_setupDimensions() {
if (!this.tty_width || !this.tty_height) {
throw new Error("DocumentBuilder doesn't have the TTY dimensions");
}
// A frame is 'taller' than the TTY because of the special UTF8 half-block
this.frame_width = this.tty_width;
// trick. Also we need to reserve 2 lines at the top for the tabs and URL bar.
this.frame_height = (this.tty_height - 2) * 2;
}
// Note how we have to keep track of 2 rows of pixels in order to create 1 row of
// the terminal.
_buildPixel(x, y) {
const colour = this.getScaledPixelAt(x, y);
if (this._bg_row.length < this.frame_width) {
this._bg_row.push(colour);
} else {
this._fg_row.push(colour);
}
if (this._fg_row.length === this.frame_width) {
this._buildTtyRow(this._bg_row, this._fg_row, y);
this.frame = this.frame.concat(this._row);
this._bg_row = [];
this._fg_row = [];
}
}
// This is where we implement the UTF8 half-block trick.
// This is a half-black: ▄ Notice how it takes up precisely half a text cell. This
// means that we can get 2 pixel colours from it, the top pixel comes from setting
// the background colour and the bottom pixel comes from setting the foreground
// colour, namely the colour of the text.
// However we can't just write random pixels to a TTY screen, we must collate 2 rows
// of native pixels for every row of the terminal.
_buildTtyRow(bg_row, fg_row, y) {
let tty_index, padding, char;
this._row = [];
const tty_row = parseInt(y / 2);
for (let x = 0; x < this.frame_width; x++) {
tty_index = (tty_row * this.frame_width) + x;
if (this._doesCellHaveACharacter(tty_index)) {
this._addCharacter(tty_index);
char = this.tty_grid[tty_index][0]
padding = this._calculateCharWidthPadding(char);
for (let p = 0; p < padding; p++) {
x++;
this._addCharacter(tty_index, ' ');
}
} else {
this._addGraphicsBlock(x, fg_row, bg_row);
}
}
}
_addCharacter(tty_index, padding = false) {
const cell = this.tty_grid[tty_index];
let char = padding ? padding : cell[0];
const fg = cell[1];
const bg = cell[2];
if (this.is_graphics_mode) {
this._row = this._row.concat(utils.ttyCell(fg, bg, char));
} else {
// TODO: Somehow communicate clickable text
this._row = this._row.concat(utils.ttyPlainCell(char));
}
}
_addGraphicsBlock(x, fg_row, bg_row) {
if (this.is_graphics_mode) {
this._row = this._row.concat(utils.ttyCell(fg_row[x], bg_row[x], '▄'));
} else {
this._row = this._row.concat(utils.ttyPlainCell(' '));
}
}
// Deal with UTF8 characters that take up more than a single cell in the TTY.
// TODO:
// 1. Do all terminals deal with wide characters the same?
// 2. Use CSS or JS so that wide characters actually flow in the DOM as 2
// monospaced characters. This will allow pages of nothing but wide
// characters to properly display.
_calculateCharWidthPadding(char) {
return charWidthInTTY(char) - 1;
}
// We need to know this because we want all empty cells to be 'transparent'
_doesCellHaveACharacter(index) {
if (this.tty_grid[index] === undefined) return false;
const char = this.tty_grid[index][0];
const is_undefined = char === undefined;
const is_empty = char === '';
const is_space = /^\s+$/.test(char);
const is_not_worth_printing = is_empty || is_space || is_undefined;
return !is_not_worth_printing;
}
}

167
webext/src/dom/frame_builder.js Normal file
View File

@ -0,0 +1,167 @@
import utils from 'utils';
import CommonMixin from 'dom/common_mixin';
import GraphicsBuilder from 'dom/graphics_builder';
import TextBuilder from 'dom/text_builder';
// Takes the graphics and text from the current DOM, combines them, then
// sends it to the background process where the rest of the UI, like tabs,
// address bar, etc will be added.
export default class extends utils.mixins(CommonMixin) {
constructor(channel, dimensions) {
super();
this.channel = channel;
this.is_graphics_mode = true;
this.dimensions = dimensions;
this.dimensions.update();
this.graphics_builder = new GraphicsBuilder(channel, dimensions);
this.text_builder = new TextBuilder(channel, dimensions, this.graphics_builder);
}
makeFrame() {
this.dimensions.update();
this._compileFrame();
this._buildFrame();
this._is_first_frame_finished = true;
}
// The user clicks on a TTY grid which has a significantly lower resolution than the
// actual browser window. So we scale the coordinates up as if the user clicked on the
// the central "pixel" of a TTY cell.
//
// Furthermore if the TTY click is on a readable character then the click is proxied
// to the original position of the character before TextBuilder snapped the character into
// position.
getDOMCoordsFromMouseCoords(x, y) {
let dom_x, dom_y, char, original_position;
const index = (y * this.dimensions.frame.width) + x;
if (this.text_builder.tty_grid.cells[index] !== undefined) {
char = this.text_builder.tty_grid.cells[index].rune;
} else {
char = false;
}
if (!char || char === '▄') {
dom_x = (x * this.dimensions.char.width);
dom_y = (y * this.dimensions.char.height);
} else {
// Recall that text can be shifted from its original position in the browser in order
// to snap it consistently to the TTY grid.
original_position = this.text_builder.tty_grid.cells[index].dom_coords;
dom_x = original_position.x;
dom_y = original_position.y;
}
return [
dom_x + (this.dimensions.char.width / 2),
dom_y + (this.dimensions.char.height / 2)
];
}
_compileFrame() {
if (!this._is_first_frame_finished || this.dimensions.dom.is_new) {
this.buildText();
}
this.graphics_builder.getScaledScreenshot();
}
// All the processes necessary to build the text for a frame. It can be CPU intensive
// so we don't want to do it all the time.
buildText() {
this.text_builder.fixJustifiedText();
this.graphics_builder.getScreenshotWithText();
this.graphics_builder.getScreenshotWithoutText();
this.text_builder.buildFormattedText();
}
_buildFrame() {
this.logPerformance(() => {
this.__buildFrame();
}, 'build frame');
}
__buildFrame() {
this.frame = [];
this._bg_row = [];
this._fg_row = [];
for (let y = 0; y < this.dimensions.frame.height; y++) {
for (let x = 0; x < this.dimensions.frame.width; x++) {
this._buildPixel(x, y);
}
}
}
// Note how we have to keep track of 2 rows of pixels in order to create 1 row of
// the terminal.
_buildPixel(x, y) {
const colour = this.graphics_builder.getScaledPixelAt(x, y);
if (this._bg_row.length < this.dimensions.frame.width) {
this._bg_row.push(colour);
} else {
this._fg_row.push(colour);
}
if (this._fg_row.length === this.dimensions.frame.width) {
this._buildTTYRow(parseInt(y / 2));
this.frame = this.frame.concat(this._row);
this._bg_row = [];
this._fg_row = [];
}
}
// This is where we implement the UTF8 half-block trick.
// This is a half-black: "▄", notice how it takes up precisely half a text cell. This
// means that we can get 2 pixel colours from it, the top pixel comes from setting
// the background colour and the bottom pixel comes from setting the foreground
// colour, namely the colour of the text.
//
// However we can't just write these "pixels" to a TTY screen, we must collate 2 rows
// of "pixels" for every row of the terminal and then we can render them together.
_buildTTYRow(row_number) {
this._row = [];
const width = this.dimensions.frame.width
for (this._current_col = 0; this._current_col < width; this._current_col++) {
this._cell = this.text_builder.tty_grid.getCellAt(this._current_col, row_number);
if (this._cell === undefined || this._cell.isTransparent()) {
this._addGraphicsBlock();
} else {
this._handleCharacter();
}
}
}
_handleCharacter() {
const padding_count = this._cell.calculateCharWidthPadding();
if (padding_count > 0) {
for (let p = 0; p < padding_count; p++) {
this._current_column++;
this._addCharacter(' ');
}
} else {
this._addCharacter();
}
}
_addCharacter(override_rune = false) {
let char;
if (override_rune) {
char = override_rune
} else {
char = this._cell.rune
}
if (this.is_graphics_mode) {
this._row = this._row.concat(
utils.ttyCell(this._cell.fg_colour, this._cell.bg_colour, char)
);
} else {
// TODO: Somehow communicate clickable text
this._row = this._row.concat(utils.ttyPlainCell(char));
}
}
_addGraphicsBlock() {
const x = this._current_col;
if (this.is_graphics_mode) {
this._row = this._row.concat(utils.ttyCell(this._fg_row[x], this._bg_row[x], '▄'));
} else {
this._row = this._row.concat(utils.ttyPlainCell(' '));
}
}
}

80
webext/src/dom/graphics_builder_mixin.js → webext/src/dom/graphics_builder.js
View File

@ -1,33 +1,45 @@
import utils from 'utils';
import CommonMixin from 'dom/common_mixin';
// Converts an instance of the visible DOM into an array of pixel values.
// Note that it does this both with and without the text visible in order
// to aid in a clean separation of the graphics and text in the final frame
// rendered in the terminal.
export default (MixinBase) => class extends MixinBase {
constructor() {
export default class extends utils.mixins(CommonMixin) {
constructor(channel, dimensions) {
super();
this.channel = channel;
this.dimensions = dimensions;
this._off_screen_canvas = document.createElement('canvas');
this._ctx = this._off_screen_canvas.getContext('2d');
this._updateCurrentViewportDimensions();
}
getPixelsAt(x, y) {
// With full-block single-glyph font on
getUnscaledFGPixelAt(x, y) {
const pixel_data_start = parseInt(
(y * (this.viewport.width * 4)) + (x * 4)
(y * this.dimensions.dom.width * 4) + (x * 4)
);
let fg_rgb = this.pixels_with_text.slice(
pixel_data_start, pixel_data_start + 3
);
return [fg_rgb[0], fg_rgb[1], fg_rgb[2]];
}
// Without any text showing at all
getUnscaledBGPixelAt(x, y) {
const pixel_data_start = parseInt(
(y * this.dimensions.dom.width * 4) + (x * 4)
);
let bg_rgb = this.pixels_without_text.slice(
pixel_data_start, pixel_data_start + 3
);
return [
[fg_rgb[0], fg_rgb[1], fg_rgb[2]],
[bg_rgb[0], bg_rgb[1], bg_rgb[2]]
]
return [bg_rgb[0], bg_rgb[1], bg_rgb[2]];
}
// Scaled to so the size where each pixel is the same size as a TTY cell
getScaledPixelAt(x, y) {
const pixel_data_start = (y * this.frame_width * 4) + (x * 4);
const pixel_data_start = (y * this.dimensions.frame.width * 4) + (x * 4);
const rgb = this.scaled_pixels.slice(pixel_data_start, pixel_data_start + 3);
return [rgb[0], rgb[1], rgb[2]];
}
@ -66,7 +78,6 @@ export default (MixinBase) => class extends MixinBase {
this._scaleCanvas();
this.scaled_pixels = this._getScreenshot();
this._unScaleCanvas();
this._is_first_frame_finished = true;
return this.scaled_pixels;
}
@ -93,36 +104,16 @@ export default (MixinBase) => class extends MixinBase {
}
_getScreenshot() {
this._updateCurrentViewportDimensions()
this.dimensions.update()
return this._getPixelData();
}
// Deal with page scrolling and other viewport changes.
// Perhaps the window has been resized to better accommodate text-sizing, or to try
// to trigger some mobile responsive CSS.
_updateCurrentViewportDimensions() {
this.viewport = {
x_scroll: window.scrollX,
y_scroll: window.scrollY,
width: window.innerWidth,
height: window.innerHeight
}
if (!this._is_scaled) {
// Resize our canvas to match the viewport. I guess this makes for efficient
// use of memory?
this._off_screen_canvas.width = this.viewport.width;
this._off_screen_canvas.height = this.viewport.height;
}
}
// Scale the screenshot so that 1 pixel approximates half a TTY cell.
_scaleCanvas() {
this._is_scaled = true;
const scale_x = this.frame_width / this.viewport.width;
const scale_y = this.frame_height / this.viewport.height;
this._hideText();
this._ctx.save();
this._ctx.scale(scale_x, scale_y);
this._ctx.scale(this.dimensions.scale_factor.width, this.dimensions.scale_factor.height);
}
_unScaleCanvas() {
@ -131,24 +122,29 @@ export default (MixinBase) => class extends MixinBase {
this._is_scaled = false;
}
_updateCanvasSize() {
if (this._is_scaled) return;
this._off_screen_canvas.width = this.dimensions.dom.width;
this._off_screen_canvas.height = this.dimensions.dom.height;
}
// Get an array of RGB values.
// This is Firefox-only. Chrome has a nicer MediaStream for this.
_getPixelData() {
let width, height;
let background_colour = 'rgb(255,255,255)';
if (this._is_scaled) {
width = this.frame_width;
height = this.frame_height;
width = this.dimensions.frame.width;
height = this.dimensions.frame.height;
} else {
width = this.viewport.width;
height = this.viewport.height;
width = this.dimensions.dom.width;
height = this.dimensions.dom.height;
}
this._updateCanvasSize();
this._ctx.drawWindow(
window,
this.viewport.x_scroll,
this.viewport.y_scroll,
this.viewport.width,
this.viewport.height,
window, 0, 0,
this.dimensions.dom.width,
this.dimensions.dom.height,
background_colour
);
return this._ctx.getImageData(0, 0, width, height).data;

145
webext/src/dom/manager.js
View File

@ -1,24 +1,28 @@
import utils from 'utils';
import CommonMixin from 'dom/common_mixin';
import DocumentBuilder from 'dom/document_builder';
import Dimensions from 'dom/dimensions';
import FrameBuilder from 'dom/frame_builder';
// Entrypoint for managing a single tab
export default class extends utils.mixins(CommonMixin) {
constructor() {
super();
// ID for element we place in the DOM to measure the size of a single monospace
// character.
this._measuring_box_id = 'browsh_em_measuring_box';
this.dimensions = new Dimensions();
this._setupInit();
}
sendFrame() {
this.document_builder.makeFrame();
this.frame_builder.makeFrame();
this._sendTabInfo();
if (!this._is_first_frame_finished) {
this.sendMessage('/status,parsing_complete');
}
this.sendMessage(`/frame,${JSON.stringify(this.document_builder.frame)}`);
if (this.frame_builder.frame.length > 0) {
this.sendMessage(`/frame,${JSON.stringify(this.frame_builder.frame)}`);
} else {
this.log("Not sending empty frame");
}
this._is_first_frame_finished = true;
}
@ -33,7 +37,7 @@ export default class extends utils.mixins(CommonMixin) {
}
_isWindowAlreadyLoaded() {
return !!this._findMeasuringBox();
return !!this.dimensions.findMeasuringBox();
}
_init(delay = 0) {
@ -65,12 +69,15 @@ export default class extends utils.mixins(CommonMixin) {
_postCommsInit() {
this.log('Webextension postCommsInit()');
this.document_builder = new DocumentBuilder(this.channel)
this.dimensions.channel = this.channel;
this.frame_builder = new FrameBuilder(this.channel, this.dimensions);
this._sendTabInfo();
this.sendMessage('/status,page_init');
this._calculateMonospaceDimensions();
this._requestInitialTTYSize();
this._listenForBackgroundMessages();
this._startWindowEventListeners()
}
_startWindowEventListeners() {
window.addEventListener("unload", () => {
this.sendMessage('/status,window_unload')
});
@ -100,13 +107,8 @@ export default class extends utils.mixins(CommonMixin) {
case '/request_frame':
this.sendFrame();
break;
case '/tty_size':
this.document_builder.tty_width = parseInt(parts[1]);
this.document_builder.tty_height = parseInt(parts[2]);
this.log(
`Tab received TTY size: ` +
`${this.document_builder.tty_width}x${this.document_builder.tty_height}`
);
case '/rebuild_text':
this._buildText();
break;
case '/stdin':
input = JSON.parse(utils.rebuildArgsToSingleArg(parts));
@ -136,16 +138,16 @@ export default class extends utils.mixins(CommonMixin) {
_handleSpecialKeys(input) {
switch (input.key) {
case 257: // up arow
window.scrollBy(0, -2 * this.document_builder.char_height);
window.scrollBy(0, -2 * this.dimensions.char.height);
break;
case 258: // down arrow
window.scrollBy(0, 2 * this.document_builder.char_height);
window.scrollBy(0, 2 * this.dimensions.char.height);
break;
case 266: // page up
window.scrollBy(0, -this.document_builder.tty_height * this.document_builder.char_height);
window.scrollBy(0, -window.innerHeight);
break;
case 267: // page down
window.scrollBy(0, this.document_builder.tty_height * this.document_builder.char_height);
window.scrollBy(0, window.innerHeight);
break;
case 18: // CTRL+r
window.location.reload();
@ -157,7 +159,8 @@ export default class extends utils.mixins(CommonMixin) {
switch (input.char) {
case 'M':
if (input.mod === 4) {
this.document_builder.is_graphics_mode = !this.document_builder.is_graphics_mode;
this.frame_builder.is_graphics_mode = !this.frame_builder.is_graphics_mode;
this.frame_builder.buildText();
}
break;
}
@ -166,10 +169,10 @@ export default class extends utils.mixins(CommonMixin) {
_handleMouse(input) {
switch (input.button) {
case 256: // scroll up
window.scrollBy(0, -20);
window.scrollBy(0, -2);
break;
case 512: // scroll down
window.scrollBy(0, 20);
window.scrollBy(0, 2);
break;
case 1: // mousedown
this._mouseAction('click', input.mouse_x, input.mouse_y);
@ -182,100 +185,20 @@ export default class extends utils.mixins(CommonMixin) {
}
_mouseAction(type, x, y) {
const [dom_x, dom_y] = this._getDOMCoordsFromMouseCoords(x, y);
const element = document.elementFromPoint(dom_x, dom_y);
const [dom_x, dom_y] = this.frame_builder.getDOMCoordsFromMouseCoords(x, y);
const element = document.elementFromPoint(
dom_x - window.scrollX,
dom_y - window.scrollY
);
const event = new MouseEvent(type, {
bubbles: true,
cancelable: true,
clientX: dom_x,
clientY: dom_y
pageX: dom_x,
pageY: dom_y
});
element.dispatchEvent(event);
}
// The user clicks on a TTY grid which has a significantly lower resolution than the
// actual browser window. So we scale the coordinates up as if the user clicked on the
// the central "pixel" of a TTY cell.
_getDOMCoordsFromMouseCoords(x, y) {
let dom_x, dom_y, char, original_position;
y = y - 2; // Because of the UI header bar
const index = (y * this.tty_width) + x;
if (this.document_builder.tty_grid[index] !== undefined) {
char = this.document_builder.tty_grid[index][0];
} else {
char = false;
}
if (!char || char === '▄') {
dom_x = (x * this.document_builder.char_width);
dom_y = (y * this.document_builder.char_height);
} else {
// Recall that text can be shifted from its original position in the browser in order
// to snap it consistently to the TTY grid.
original_position = this.document_builder.tty_grid[index][4];
dom_x = original_position.x;
dom_y = original_position.y;
}
return [
dom_x + (this.document_builder.char_width / 2),
dom_y + (this.document_builder.char_height / 2)
];
}
// The background process can't send the TTY size as soon as it gets it because maybe
// the a tab doesn't exist yet. So we request it ourselves - because we'd have to be
// ready in order to request.
_requestInitialTTYSize() {
this.sendMessage('/request_tty_size');
}
// This is critical in order for the terminal to match the browser as closely as possible.
// Ideally we want the browser's window size to be exactly multiples of the terminal's
// dimensions. So if the terminal is 80x40 and the font-size is 12px (12x6 pixels), then
// the window should be 480x480. Also knowing the precise font-size helps the text builder
// map un-snapped text to the best grid cells - grid cells that represent the terminal's
// character positions.
// The reason that we can't just do some basic maths on the CSS `font-size` value we enforce
// is that there are various factors that can skew the actual font dimensions on the page.
// For instance, you can't guarantee that a browser is using exactly the same version of
// a named monospace font. Also different browser families and even different versions of
// the same browser may have subtle differences in how they render text. Furthermore we can
// actually get floating point accuracy if we use `Element.getBoundingClientRect()` which
// further helps as calculations are compounded during our rendering processes.
_calculateMonospaceDimensions() {
const element = this._getOrCreateMeasuringBox();
const dom_rect = element.getBoundingClientRect();
this.document_builder.char_width = dom_rect.width;
this.document_builder.char_height = dom_rect.height + 2; // TODO: WTF is this magic number?
this.sendMessage(
`/char_size,` +
`${this.document_builder.char_width},` +
`${this.document_builder.char_height}`
);
this.log(
`Tab char dimensions: ` +
`${this.document_builder.char_width}x${this.document_builder.char_height}`
);
}
// Back when printing was done by physical stamps, it was convention to measure the
// font-size using the letter 'M', thus where we get the unit 'em' from. Not that it
// should make any difference to us, but it's nice to keep a tradition.
_getOrCreateMeasuringBox() {
let measuring_box = this._findMeasuringBox();
if (measuring_box) return measuring_box;
measuring_box = document.createElement('span');
measuring_box.id = this._measuring_box_id;
measuring_box.style.visibility = 'hidden';
var M = document.createTextNode('M');
measuring_box.appendChild(M);
document.body.appendChild(measuring_box);
return measuring_box;
}
_findMeasuringBox() {
return document.getElementById(this._measuring_box_id);
}
_sendTabInfo() {
const title_object = document.getElementsByTagName("title");
let info = {

308
webext/src/dom/text_builder.js Normal file
View File

@ -0,0 +1,308 @@
import _ from 'lodash';
import utils from 'utils';
import CommonMixin from 'dom/common_mixin';
import TTYCell from 'dom/tty_cell';
import TTYGrid from 'dom/tty_grid';
// Convert the text on the page into a snapped 2-dimensional grid to be displayed directly
// in the terminal.
export default class extends utils.mixins(CommonMixin) {
constructor(channel, dimensions, graphics_builder) {
super();
this.channel = channel;
this.dimensions = dimensions;
this.tty_grid = new TTYGrid(dimensions, graphics_builder);
this._parse_started_elements = [];
// A `range` is the DOM's representation of elements and nodes as they are rendered in
// the DOM. Think of the 'range' that is created when you select/highlight text for
// copy-pasting, those usually blue-ish rectangles around the selected text are ranges.
this._range = document.createRange();
}
buildFormattedText() {
this._updateState();
this._getTextNodes();
this._positionTextNodes();
}
_updateState() {
this.tty_grid.cells = [];
this._parse_started_elements = [];
}
// This is relatively cheap: around 50ms for a 13,000 word Wikipedia page
_getTextNodes() {
this.logPerformance(() => {
this.__getTextNodes();
}, 'tree walker');
}
// This should be around ?? for a largish Wikipedia page of 13,000 words
_positionTextNodes() {
this.logPerformance(() => {
this.__positionTextNodes();
}, 'position text nodes');
}
// Search through every node in the DOM looking for displayable text.
__getTextNodes() {
this._text_nodes = [];
const walker = document.createTreeWalker(
document.body,
NodeFilter.SHOW_TEXT,
{ acceptNode: this._isRelevantTextNode },
false
);
while(walker.nextNode()) this._text_nodes.push(walker.currentNode);
}
// Does the node contain text that we want to display?
_isRelevantTextNode(node) {
// Ignore nodes with only whitespace
if (/^\s+$/.test(node.textContent) || node.textContent === '') {
return NodeFilter.FILTER_REJECT;
}
return NodeFilter.FILTER_ACCEPT;
}
__positionTextNodes() {
for (const node of this._text_nodes) {
this._node = node
this._text = node.textContent
this._formatText();
this._character_index = 0;
this._positionSingleTextNode();
}
}
_formatText() {
this._normaliseWhitespace()
}
// Justified text uses the space between words to stretch a line to perfectly fit from
// end to end. That'd be ok if it only stretched by exact units of monospace width, but
// it doesn't, which messes with our fragile grid system.
// TODO:
// * It'd be nice to detect right-justified text so we can keep it. Just need to be
// careful with things like traversing parents up the DOM, or using `computedStyle()`
// because they can be expensive.
// * Another approach could be to explore how a global use of `pre` styling renders
// pages.
// * Also, is it possible and/or faster to do this once in the main style sheet? Or
// even by a find-replace on all occurrences of 'justify'?
// * Yet another thing, the style change doesn't actually get picked up until the
// next frame. Thus why the loop is independent of the `positionTextNodes()` loop.
fixJustifiedText() {
this._getTextNodes();
for (const node of this._text_nodes) {
node.parentElement.style.textAlign = 'left';
}
}
// The need for this wasn't immediately obvious to me. The fact is that the DOM stores
// text nodes _as they are written in the HTML doc_. Therefore, if you've written some
// nicely indented HTML, then the text node will actually contain those as something like
// `\n text starts here`
// It's just that the way CSS works most of the time means that whitespace is collapsed
// so viewers never notice.
//
// TODO:
// The normalisation here of course destroys the formatting of `white-space: pre`
// styling, like code snippets for example. So hopefully we can detect the node's
// `white-space` setting and skip this function if necessary?
_normaliseWhitespace() {
// Unify all whitespace to a single space character
this._text = this._text.replace(/[\t\n\r ]+/g, " ");
if (this._isFirstParseInElement()) {
// Remove whitespace at the beginning
if (this._text.charAt(0) === " ") {
this._text = this._text.substring(1, this._text.length);
}
}
// Remove whitespace at the end
if (this._text.charAt(this._text.length - 1) === " ") {
this._text = this._text.substring(0, this._text.length - 1);
}
}
// Knowing if a text node is the first within its parent element helps to decide
// whether to remove its leading whitespace or not.
//
// An element may contain many text nodes. For example a `<p>` element may contain a
// starting text node followed by a `<a>` tag, finishing with another plain text node. We
// only want to remove leading whitespace from the text at the _beginning_ of a line.
// Usually we can do this just by checking if a DOM rectangle's position is further down
// the page than the previous one - but of course there is nothing to compare the first
// DOM rectangle to. What's more, DOM rects are grouped per _text node_, NOT per element
// and we are not guaranteed to iterate through elements in the order that text flows.
// Therefore we need to make the assumption that plain text nodes flow within their shared
// parent element. There is a possible caveat here for elements starting with another
// element (like a link), where that sub-element contains leading whitespace.
_isFirstParseInElement() {
let element = this._node.parentElement;
const is_parse_started = _.includes(this._parse_started_elements, element);
if (is_parse_started) {
return false
} else {
this._parse_started_elements.push(element);
return true
}
}
// Here is where we actually make use of the rather strict monospaced and fixed font size
// CSS rules enforced by the webextension. Of course the CSS is never going to be able to
// perfectly snap characters onto a grid, so we force it here instead. At least we can be
// fairly certain that every character at least takes up the same space as a TTY cell, it
// just might not be perfectly aligned. So here we just round down all coordinates to force
// the snapping.
//
// Use `this.addClientRectsOverlay(dom_rects, text);` to see DOM rectangle outlines in a
// real browser.
_positionSingleTextNode() {
this._dom_box = {};
this._previous_dom_box = {};
for (const dom_box of this._getNodeDOMBoxes()) {
this._dom_box.top = dom_box.top;
this._dom_box.left = dom_box.left;
this._dom_box.width = dom_box.width;
this._handleSingleDOMBox()
this._previous_dom_box = _.clone(this._dom_box);
}
}
// This is the key to being able to display formatted text within the strict confines
// of a TTY. DOM Rectangles are closely related to selection ranges (like when you click
// and drag the mouse cursor over text). Think of an individual DOM rectangle as a single
// bar of highlighted selection. So that, for example, a 3 line paragraph will have 3
// DOM rectangles. Fortunately DOMRect coordinates and dimensions are precisely defined.
// Although do note that, unlike selection ranges, sub-selections can appear seemingly
// inside other selections for things like italics or anchor tags.
_getNodeDOMBoxes() {
this._range.selectNode(this._node);
return this._range.getClientRects()
}
// A single box is always a valid rectangle. Therefore a single box will, for example,
// never straddle 2 lines as there is no guarantee that a valid rectangle can be formed.
// We can use this to our advantage by stepping through coordinates of a box to get the
// exact position of every single individual character. We just have to understand and
// follow exactly how the DOM flows text - easier said than done.
_handleSingleDOMBox() {
this._prepareToParseDOMBox();
for (let step = 0; step < this._tty_box.width; step++) {
this._handleSingleCharacter();
this._stepToNextCharacter();
}
}
_prepareToParseDOMBox() {
this._convertDOMBoxToAbsoluteCoords()
this._createSyncedTTYBox();
this._createTrackers()
this._ignoreUnrenderedWhitespace();
this._setCurrentCharacter();
}
_setCurrentCharacter() {
// Note that it's possible for text to straddle many DOM boxes
this._current_character = this._text.charAt(this._character_index);
}
// Everything hinges on these 2 trackers being in sync. The DOM tracker is defined by
// actual pixel coordinates and we move horizontally, from left to right, each step
// being the width of a single character. The TTY tracker moves in the same way except
// each step is a new single cell within the TTY.
_createTrackers() {
this._dom_tracker = {
x: utils.snap(this._dom_box.left),
y: utils.snap(this._dom_box.top)
}
this._tty_tracker = {
x: this._tty_box.col_start,
y: this._tty_box.row
}
}
_handleSingleCharacter() {
let cell = new TTYCell();
cell.rune = this._current_character;
cell.tty_coords = _.clone(this._tty_tracker);
cell.dom_coords = _.clone(this._dom_tracker);
cell.parent_element = this._node.parentElement;
this.tty_grid.addCell(cell);
}
_stepToNextCharacter(tracked = true) {
this._character_index++;
this._setCurrentCharacter();
if (tracked) {
this._dom_tracker.x += this.dimensions.char.width;
this._tty_tracker.x++;
}
}
// There is a careful tracking between the currently parsed character of `this._text`
// and the position of the current 'cell' space within `this._dom_box`. So we must be precise
// in how we synchronise them. This requires following the DOM's method for wrapping text.
// Recall how the DOM will split a line at a space character boundry. That space character
// is then in fact never rendered - its existence is never registered within the dimensions
// of a DOM rectangle's box (`this._dom_box`).
_ignoreUnrenderedWhitespace() {
if (this._isNewLine()) {
if (/[\t\n\r ]+/.test(this._current_character)) this._stepToNextCharacter(false);
}
}
// Is the current DOM rectangle further down the page than the previous?
_isNewLine() {
if (this._previous_dom_box === {}) return false;
return this._dom_box.top > this._previous_dom_box.top
}
// The DOM returns box coordinates relative to the viewport. As we are rendering the
// entire DOM as a single frame, then we need the coords to be relative to the top-left
// of the DOM itself.
_convertDOMBoxToAbsoluteCoords() {
this._dom_box.left += this.dimensions.dom.x_scroll;
this._dom_box.top += this.dimensions.dom.y_scroll;
}
// Round and snap a DOM rectangle as if it were placed in the TTY frame
_createSyncedTTYBox() {
this._tty_box = {
col_start: utils.snap(this._dom_box.left / this.dimensions.char.width),
row: utils.snap(this._dom_box.top / this.dimensions.char.height),
width: utils.snap(this._dom_box.width / this.dimensions.char.width),
}
}
// Purely for debugging.
//
// Draws a red border around all the DOMClientRect nodes.
// Based on code from the MDN docs site.
_addClientRectsOverlay(dom_rects, normalised_text) {
// Don't draw on every frame
if (this.is_first_frame_finished) return;
// Absolutely position a div over each client rect so that its border width
// is the same as the rectangle's width.
// Note: the overlays will be out of place if the user resizes or zooms.
for (const rect of dom_rects) {
let tableRectDiv = document.createElement('div');
// A DOMClientRect object only contains dimensions, so there's no way to identify it
// to a node, so let's put its text as an attribute so we can cross-check if needs be.
tableRectDiv.setAttribute('browsh-text', normalised_text);
let tty_row = parseInt(Math.round(rect.top / this.dimemnsions.char.height));
tableRectDiv.setAttribute('tty_row', tty_row);
tableRectDiv.style.position = 'absolute';
tableRectDiv.style.border = '1px solid red';
tableRectDiv.style.margin = tableRectDiv.style.padding = '0';
tableRectDiv.style.top = rect.top + 'px';
tableRectDiv.style.left = rect.left + 'px';
// We want rect.width to be the border width, so content width is 2px less.
tableRectDiv.style.width = (rect.width - 2) + 'px';
tableRectDiv.style.height = (rect.height - 2) + 'px';
document.body.appendChild(tableRectDiv);
}
}
}

340
webext/src/dom/text_builder_mixin.js
View File

@ -1,340 +0,0 @@
import _ from 'lodash';
// Convert the text on the page into a snapped 2-dimensional grid to be displayed directly
// in the terminal.
export default (MixinBase) => class extends MixinBase {
constructor() {
super();
this._parse_started_elements = [];
}
buildFormattedText() {
this._updateState();
this._getTextNodes();
this._positionTextNodes();
this._is_first_frame_finished = true;
}
_updateState() {
this.tty_grid = [];
this.tty_dom_width = this.tty_width;
// For Tabs and URL bar.
this.tty_dom_height = this.tty_height - 2;
this._parse_started_elements = [];
}
// This is relatively cheap: around 50ms for a 13,000 word Wikipedia page
_getTextNodes() {
this.logPerformance(() => {
this.__getTextNodes();
}, 'tree walker');
}
// This should be around 125ms for a largish Wikipedia page of 13,000 words
_positionTextNodes() {
this.logPerformance(() => {
this.__positionTextNodes();
}, 'position text nodes');
}
// Search through every node in the DOM looking for displayable text.
__getTextNodes() {
this._text_nodes = [];
const walker = document.createTreeWalker(
document.body,
NodeFilter.SHOW_TEXT,
{ acceptNode: this._isRelevantTextNode },
false
);
while(walker.nextNode()) this._text_nodes.push(walker.currentNode);
}
// Does the node contain text that we want to display?
_isRelevantTextNode(node) {
// Ignore nodes with only whitespace
if (/^\s+$/.test(node.textContent) || node.textContent === '') {
return NodeFilter.FILTER_REJECT;
}
return NodeFilter.FILTER_ACCEPT;
}
// This is the key to being able to display formatted text within the strict confines
// of a TTY. DOM Rectangles are closely related to selection ranges (like when you click
// and drag the mouse cursor over text). Think of an individual DOM rectangle as a single
// bar of highlighted selection. So that, for example, a 3 line paragraph will have 3
// DOM rectangles. Fortunately DOMRect coordinates and dimensions are precisely defined.
// Although do note that, unlike selection ranges, sub-selections can appear seemingly
// inside other selections for things like italics or anchor tags.
__positionTextNodes() {
let range = document.createRange();
let bounding_box;
for (const node of this._text_nodes) {
range.selectNode(node);
bounding_box = range.getBoundingClientRect();
if (this._isBoxOutsideViewport(bounding_box)) continue;
this._fixJustifiedText(node);
this._formatTextForTTYGrid(
this._normaliseWhitespace(node.textContent, node.parentElement),
range.getClientRects(),
node.parentElement
);
}
}
// If even a single pixel is inside the viewport we need to check it
_isBoxOutsideViewport(bounding_box) {
const is_top_in =
bounding_box.top >= 0 &&
bounding_box.top < this.viewport.height;
const is_bottom_in =
bounding_box.bottom >= 0 &&
bounding_box.bottom < this.viewport.height;
const is_left_in =
bounding_box.left >= 0 &&
bounding_box.left < this.viewport.width;
const is_right_in =
bounding_box.right >= 0 &&
bounding_box.right < this.viewport.width;
return !((is_top_in || is_bottom_in) && (is_left_in || is_right_in));
}
// Justified text uses the space between words to stretch a line to perfectly fit from
// end to end. That'd be ok if it only stretched by exact units of monospace width, but
// it doesn't, which messes with our fragile grid system.
// TODO:
// * It'd be nice to detect right-justified text so we can keep it. Just need to be
// careful with things like traversing parents up the DOM, or using `computedStyle()`
// because they can be expensive.
// * Another approach could be to explore how a global use of `pre` styling renders
// pages.
// * Also, is it possible and/or faster to do this once in the main style sheet? Or
// even by a find-replace on all occurrences of 'justify'?
// * Yet another thing, the style change doesn't actually get picked up until the
// next frame.
_fixJustifiedText(node) {
node.parentElement.style.textAlign = 'left';
}
// The need for this wasn't immediately obvious to me. The fact is that the DOM stores
// text nodes _as they are written in the HTML doc_. Therefore, if you've written some
// nicely indented HTML, then the text node will actually contain those as something like
// `\n text starts here`
// It's just that the way CSS works most of the time means that whitespace is collapsed
// so viewers never notice.
//
// TODO:
// The normalisation here of course destroys the formatting of `white-space: pre`
// styling, like code snippets for example. So hopefully we can detect the node's
// `white-space` setting and skip this function if necessary?
_normaliseWhitespace(text, parent) {
text = text.replace(/[\t\n\r ]+/g, " ");
if (this._isFirstParseInElement(parent)) {
if (text.charAt(0) === " ") text = text.substring(1, text.length);
}
if (text.charAt(text.length - 1) === " ") text = text.substring(0, text.length - 1);
return text;
}
// An element may contain many text nodes. For example a `<p>` element may contain a
// starting text node followed by a `<a>` tag, finishing with another plain text node. We
// only want to remove leading whitespace from the text at the _beginning_ of a line.
// Usually we can do this just by checking if a DOM rectangle's position is further down
// the page than the previous one - but of course there is nothing to compare the first
// DOM rectangle to. What's more, DOM rects are grouped per _text node_, NOT per element
// and we are not guaranteed to iterate through elements in the order that text flows.
// Therefore we need to make the assumption that plain text nodes flow within their shared
// parent element. There is a possible caveat here for elements starting with another
// element (like a link), where that sub-element contains leading whitespace.
_isFirstParseInElement(element) {
const is_parse_started = _.includes(this._parse_started_elements, element);
if (is_parse_started) {
return false
} else {
this._parse_started_elements.push(element);
return true
}
}
// Here is where we actually make use of the rather strict monospaced and fixed font size
// CSS rules enforced by the webextension. Of course the CSS is never going to be able to
// perfectly snap characters onto a grid, so we force it here instead. At least we can be
// fairly certain that every character at least takes up the same space as a TTY cell, it
// just might not be perfectly aligned. So here we just round down all coordinates to force
// the snapping.
// Use `this.addClientRectsOverlay(dom_rects, text);` to see DOM rectangle outlines in a
// real browser.
_formatTextForTTYGrid(text, dom_rects, parent_element) {
let col, tty_box, step, character, previous_box, origin;
let character_index = 0;
for (const box of dom_rects) {
if (this._isBoxOutsideViewport(box)) return;
if (this._isNewLine(previous_box, box)) {
character = text.charAt(character_index);
if (/[\t\n\r ]+/.test(character)) character_index++;
}
tty_box = this._convertBoxToTTYUnits(box);
col = tty_box.col_start;
origin = {
x: parseInt(Math.round(box.left)),
y: parseInt(Math.round(box.top))
}
for (step = 0; step < tty_box.width; step++) {
character = text.charAt(character_index);
this._placeCharacterOnTTYGrid(col, tty_box.row, origin, character, parent_element);
origin.x = origin.x + this.char_width;
character_index++;
col++
}
previous_box = box;
}
}
// Is the current DOM rectangle further down the page than the previous?
_isNewLine(previous_box, current_box) {
if (previous_box === undefined) return false;
return current_box.top > previous_box.top
}
// Round and snap a DOM rectangle as if it were placed in the terminal
_convertBoxToTTYUnits(viewport_dom_rect) {
return {
col_start: this.snap(viewport_dom_rect.left / this.char_width),
row: this.snap(viewport_dom_rect.top / this.char_height),
width: this.snap(viewport_dom_rect.width / this.char_width),
}
}
_placeCharacterOnTTYGrid(col, row, original_position, character, parent_element) {
const index = (row * this.tty_dom_width) + col;
if (this._isExistingCharacter(index)) {
if (!this._isHighestLayer(index, parent_element)) return;
}
if (this._isCharOutsideGrid(col, row)) return;
const colours = this._getCharacterColours(original_position);
if (!colours) return;
if (this._isCharObscured(colours)) return;
this.tty_grid[index] = [
character,
colours[0],
colours[1],
parent_element,
_.clone(original_position)
];
}
// Don't clobber - for now at least.
// TODO: Use `getComputedStyles()` and save for the whole parent element.
_isExistingCharacter(index) {
return !!this.tty_grid[index];
}
// When a character clobbers another character in the grid, we can't use our
// text show/hide trick to know if the character is visible in the final DOM. So we have
// to use standard CSS inspection instead. Hopefully this doesn't happen often because
// it's expensive.
// TODO: Make comprehensive
_isHighestLayer(index_of_tenant, parent_element_of_challenger) {
const tenant_styles = this._getStyles(this.tty_grid[index_of_tenant][3]);
const challenger_styles = this._getStyles(parent_element_of_challenger);
if (
challenger_styles.visibility === 'hidden' ||
challenger_styles.display === 'none'
) {
return false;
}
return tenant_styles.zIndex < challenger_styles.zIndex;
}
// Get or cache the total cascaded calculated styles for an element
_getStyles(element) {
if (!element.browsh_calculated_styles) {
let styles = window.getComputedStyle(element);
element.browsh_calculated_styles = styles;
}
return element.browsh_calculated_styles;
}
// Get the colours right in the middle of the character's font. Returns both the colour
// when the text is displayed and when it's hidden.
_getCharacterColours(original_position) {
// Don't use a full half, just because it means that we can use very small mock pixel
// arrays during testing - rounding to the top-left saves having to write and extra
// column and row.
const half = 0.449;
const offset_x = this.snap(original_position.x + (this.char_width * half));
const offset_y = this.snap(original_position.y + (this.char_height * half));
if (this._isCharCentreOutsideViewport(offset_x, offset_y)) return false;
return this.getPixelsAt(offset_x, offset_y);
}
// Check if the char is in the viewport again because of x increments, y potentially
// being rounded up and of course the offset to make sure the sample is within the
// unicode block.
_isCharCentreOutsideViewport(x, y) {
if (
x >= this.viewport.width ||
x < 0 ||
y >= this.viewport.height ||
y < 0
) return false;
}
// Theoretically this should only be needed for DOM rectangles that _straddle_ the
// viewport.
_isCharOutsideGrid(col, row) {
return col >= this.tty_dom_width || row >= this.tty_dom_height;
}
// This is somewhat of a, hopefully elegant, hack. So, imagine that situation where you're
// browsing a web page and a popup appears; perhaps just a select box, or menu, or worst
// of all a dreaded full-page overlay. Now, DOM rectangles don't take into account whether
// they are the uppermost visible element, so we're left in a bit of a pickle. The only JS
// way to know if an element is visible is to use `Document.elementFromPoint(x, y)`, where
// you compare the returned element with the element whose visibility you're checking.
// This is has a number of problems. Firstly, it only checks one coordinate in the element
// for visibility, which of course isn't going to 100% reliably speak for all the
// characters in the element. Secondly, even ignoring the first caveat, running
// `elementFromPoint()` for every character is very expensive, around 25ms for an average
// DOM. So it's basically a no-go. So instead we take advantage of the fact that we're
// working with a snapshot of the the webpage's pixels. It's pretty good assumption that if
// you make the text transparent and a pixel's colour doesn't change then that character
// must be obscured by something.
// There are of course some potential edge cases with this. What if we get a false
// positive, where a character is obscured _by another character_? Hopefully in such a
// case we can work with `z-index` so that characters justifiably overwrite each other in
// the TTY grid.
_isCharObscured(colours) {
return colours[0][0] === colours[1][0] &&
colours[0][1] === colours[1][1] &&
colours[0][2] === colours[1][2];
}
// Purely for debugging. Draws a red border around all the DOMClientRect nodes.
// Based on code from the MDN docs site.
_addClientRectsOverlay(dom_rects, normalised_text) {
// Don't draw on every frame
if (this.is_first_frame_finished) return;
// Absolutely position a div over each client rect so that its border width
// is the same as the rectangle's width.
// Note: the overlays will be out of place if the user resizes or zooms.
for (const rect of dom_rects) {
let tableRectDiv = document.createElement('div');
// A DOMClientRect object only contains dimensions, so there's no way to identify it
// to a node, so let's put its text as an attribute so we can cross-check if needs be.
tableRectDiv.setAttribute('browsh-text', normalised_text);
let tty_row = parseInt(Math.round(rect.top / this.char_height));
tableRectDiv.setAttribute('tty_row', tty_row);
tableRectDiv.style.position = 'absolute';
tableRectDiv.style.border = '1px solid red';
let scrollTop = document.documentElement.scrollTop || document.body.scrollTop;
let scrollLeft = document.documentElement.scrollLeft || document.body.scrollLeft;
tableRectDiv.style.margin = tableRectDiv.style.padding = '0';
tableRectDiv.style.top = (rect.top + scrollTop) + 'px';
tableRectDiv.style.left = (rect.left + scrollLeft) + 'px';
// We want rect.width to be the border width, so content width is 2px less.
tableRectDiv.style.width = (rect.width - 2) + 'px';
tableRectDiv.style.height = (rect.height - 2) + 'px';
document.body.appendChild(tableRectDiv);
}
}
}

52
webext/src/dom/tty_cell.js Normal file
View File

@ -0,0 +1,52 @@
import charWidthInTTY from 'string-width';
// A single cell on the TTY grid
export default class {
// When a character clobbers another character in the grid, we can't use our
// text show/hide trick to know if the character is visible in the final DOM. So we have
// to use standard CSS inspection instead. Hopefully this doesn't happen often because
// it's expensive.
// TODO: Make comprehensive
isHighestLayer(challenger_cell) {
const tenant_styles = this.getStyles();
const challenger_styles = challenger_cell.getStyles();
if (
challenger_styles.visibility === 'hidden' ||
challenger_styles.display === 'none'
) {
return false;
}
return tenant_styles.zIndex > challenger_styles.zIndex;
}
// Get or cache the total cascaded calculated styles for an element
getStyles() {
let element = this.parent_element;
if (!element.browsh_calculated_styles) {
let styles = window.getComputedStyle(element);
element.browsh_calculated_styles = styles;
}
return element.browsh_calculated_styles;
}
isTransparent() {
const is_undefined = this.rune === undefined;
const is_empty = this.rune === '';
const is_space = /^\s+$/.test(this.rune);
const is_not_worth_printing = is_empty || is_space || is_undefined;
return is_not_worth_printing;
}
// Deal with UTF8 characters that take up more than a single cell in the TTY.
// Eg; 比如说
// TODO:
// 1. Do all terminals deal with wide characters the same?
// 2. Use CSS or JS so that wide characters actually flow in the DOM as 2
// monospaced characters. This will allow pages of nothing but wide
// characters to render/flow as closely as possible ot how they will appear
// in the TTY.
calculateCharWidthPadding() {
return charWidthInTTY(this.rune) - 1;
}
}

85
webext/src/dom/tty_grid.js Normal file
View File

@ -0,0 +1,85 @@
import utils from 'utils';
// The TTY grid
export default class {
constructor(dimensions, graphics_builder) {
this.dimensions = dimensions;
this.graphics_builder = graphics_builder;
}
getCell(index) {
return this.cells[index];
}
getCellAt(x, y) {
return this.cells[(y * this.dimensions.frame.width) + x];
}
addCell(new_cell) {
new_cell.index = this._calculateIndex(new_cell);
if (!this._handleExistingCell(new_cell)) return false;
if (!this._handleCellVisibility(new_cell)) return false;
this.cells[new_cell.index] = new_cell;
}
_handleExistingCell(new_cell) {
let existing_cell = this.cells[new_cell.index];
if (existing_cell !== undefined) {
if (!new_cell.isHighestLayer(existing_cell)) return false;
}
return true;
}
_handleCellVisibility(new_cell) {
const colours = this._getColours(new_cell);
if (!colours) return false;
if (this._isCharObscured(colours)) return false;
new_cell.fg_colour = colours[0];
new_cell.bg_colour = colours[1];
return true;
}
_calculateIndex(cell) {
return (cell.tty_coords.y * this.dimensions.frame.width) + cell.tty_coords.x;
}
// Get the colours right in the middle of the character's font. Returns both the colour
// when the text is displayed and when it's hidden.
_getColours(cell) {
// Don't use a full half, just because it means that we can use very small mock pixel
// arrays during testing - rounding to the top-left saves having to write an extra
// column and row.
const half = 0.449;
const offset_x = utils.snap(cell.dom_coords.x + (this.dimensions.char.width * half));
const offset_y = utils.snap(cell.dom_coords.y + (this.dimensions.char.height * half));
const fg_colour = this.graphics_builder.getUnscaledFGPixelAt(offset_x, offset_y);
const bg_colour = this.graphics_builder.getUnscaledBGPixelAt(offset_x, offset_y);
return [fg_colour, bg_colour]
}
// This is somewhat of a, hopefully elegant, hack. So, imagine that situation where you're
// browsing a web page and a popup appears; perhaps just a select box, or menu, or worst
// of all a dreaded full-page overlay. Now, DOM rectangles don't take into account whether
// they are the uppermost visible element, so we're left in a bit of a pickle. The only JS
// way to know if an element is visible is to use `Document.elementFromPoint(x, y)`, where
// you compare the returned element with the element whose visibility you're checking.
// This is has a number of problems. Firstly, it only checks one coordinate in the element
// for visibility, which of course isn't going to 100% reliably speak for all the
// characters in the element. Secondly, even ignoring the first caveat, running
// `elementFromPoint()` for every character is very expensive, around 25ms for an average
// DOM. So it's basically a no-go. So instead we take advantage of the fact that we're
// working with a snapshot of the the webpage's pixels. It's pretty good assumption that if
// you make the text transparent and a pixel's colour doesn't change then that character
// must be obscured by something.
//
// There are of course some potential edge cases with this. What if we get a false
// positive, where a character is obscured _by another character_? Hopefully in such a
// case we can work with `z-index` so that characters justifiably overwrite each other in
// the TTY grid.
_isCharObscured(colours) {
return colours[0][0] === colours[1][0] &&
colours[0][1] === colours[1][1] &&
colours[0][2] === colours[1][2];
}
}

4
webext/src/utils.js
View File

@ -17,6 +17,10 @@ export default {
return this.ttyCell(null, null, character);
},
snap: function (number) {
return parseInt(Math.round(number));
},
rebuildArgsToSingleArg: function (args) {
return args.slice(1).join(',');
}

32
webext/test/document_builder_spec.js
View File

@ -1,32 +0,0 @@
import sandbox from 'helper';
import {expect} from 'chai';
import DocumentBuilder from 'dom/document_builder';
import canvas_pixels from 'fixtures/canvas_pixels';
import tty_grid from 'fixtures/tty_grid';
describe('Document Builder', ()=> {
let document_builder;
beforeEach(()=> {
document_builder = new DocumentBuilder();
document_builder.tty_grid = tty_grid;
sandbox.stub(DocumentBuilder.prototype, '_getPixelData').returns(canvas_pixels);
sandbox.stub(DocumentBuilder.prototype, 'buildFormattedText').returns();
});
it('should merge pixels and text into a 1D array', ()=> {
document_builder.tty_width = 3;
document_builder.tty_height = 2 + 2;
document_builder.makeFrame();
const frame = document_builder.frame.join(',');
expect(frame).to.eq(
'0,0,0,111,111,111,▄,' +
'111,111,111,222,222,222,😐,' +
'0,0,0,111,111,111,▄,' +
'111,111,111,222,222,222,😄,' +
'111,111,111,0,0,0,▄,' +
'111,111,111,222,222,222,😂'
);
});
});

9
webext/test/fixtures/tty_grid.js → webext/test/fixtures/cells.js vendored
View File

@ -1,3 +1,5 @@
import TTYCell from "dom/tty_cell";
let base = build(
[
'', '😐', '',
@ -10,9 +12,14 @@ let base = build(
export default base;
function build(text, fg_colour, bg_colour) {
let cell;
let grid = [];
for(const character of text) {
grid.push([character, fg_colour, bg_colour]);
cell = new TTYCell();
cell.rune = character;
cell.fg_colour = fg_colour;
cell.bg_colour = bg_colour;
grid.push(cell);
}
return grid;
}

35
webext/test/frame_builder_spec.js Normal file
View File

@ -0,0 +1,35 @@
import sandbox from 'helper';
import {expect} from 'chai';
import Dimensions from 'dom/dimensions';
import FrameBuilder from 'dom/frame_builder';
import GraphicsBuilder from 'dom/graphics_builder';
import TextBuilder from 'dom/text_builder';
import canvas_pixels from 'fixtures/canvas_pixels';
import cells from 'fixtures/cells';
describe('Frame Builder', ()=> {
let frame_builder;
beforeEach(()=> {
global.document.documentElement.scrollHeight = 4;
frame_builder = new FrameBuilder(undefined, new Dimensions());
frame_builder.text_builder.tty_grid.cells = cells;
sandbox.stub(GraphicsBuilder.prototype, '_getPixelData').returns(canvas_pixels);
sandbox.stub(TextBuilder.prototype, 'buildFormattedText').returns();
sandbox.stub(TextBuilder.prototype, 'fixJustifiedText').returns();
});
it('should merge pixels and text into a 1D array', () => {
frame_builder.makeFrame();
const frame = frame_builder.frame.join(',');
expect(frame).to.eq(
'0,0,0,111,111,111,▄,' +
'111,111,111,222,222,222,😐,' +
'0,0,0,111,111,111,▄,' +
'111,111,111,222,222,222,😄,' +
'111,111,111,0,0,0,▄,' +
'111,111,111,222,222,222,😂'
);
});
});

39
webext/test/helper.js
View File

@ -1,22 +1,32 @@
import sinon from 'sinon';
import DocumentBuilder from 'dom/document_builder';
import Dimensions from 'dom/dimensions';
import FrameBuilder from 'dom/frame_builder';
import GraphicsBuilder from 'dom/graphics_builder';
import MockRange from 'mocks/range'
var sandbox = sinon.sandbox.create();
beforeEach(() => {
sandbox.stub(DocumentBuilder.prototype, '_hideText').returns(true);
sandbox.stub(DocumentBuilder.prototype, '_showText').returns(true);
sandbox.stub(DocumentBuilder.prototype, '_scaleCanvas').returns(true);
sandbox.stub(DocumentBuilder.prototype, '_unScaleCanvas').returns(true);
sandbox.stub(DocumentBuilder.prototype, 'sendMessage').returns(true);
sandbox.stub(Dimensions.prototype, '_getOrCreateMeasuringBox').returns(element);
sandbox.stub(GraphicsBuilder.prototype, '_hideText').returns(true);
sandbox.stub(GraphicsBuilder.prototype, '_showText').returns(true);
sandbox.stub(GraphicsBuilder.prototype, '_scaleCanvas').returns(true);
sandbox.stub(GraphicsBuilder.prototype, '_unScaleCanvas').returns(true);
sandbox.stub(FrameBuilder.prototype, 'sendMessage').returns(true);
});
afterEach(() => {
sandbox.restore();
});
global.dimensions = {
char: {
width: 1,
height: 2 - 2
}
}
global.document = {
addEventListener: () => {},
getElementById: () => {},
@ -33,9 +43,15 @@ global.document = {
getContext: () => {}
}
},
documentElement: {
scrollWidth: 3,
scrollHeight: 8
},
location: {
href: 'https://www.google.com'
}
},
scrollX: 0,
scrollY: 0
};
global.DEVELOPMENT = false;
@ -46,4 +62,13 @@ global.performance = {
now: () => {}
}
let element = {
getBoundingClientRect: () => {
return {
width: global.dimensions.char.width,
height: global.dimensions.char.height
}
}
}
export default sandbox;

112
webext/test/text_builder_spec.js
View File

@ -1,7 +1,10 @@
import sandbox from 'helper';
import { expect } from 'chai';
import DocumentBuilder from 'dom/document_builder';
import Dimensions from 'dom/dimensions';
import FrameBuilder from 'dom/frame_builder';
import GraphicsBuilder from 'dom/graphics_builder';
import TTYCell from 'dom/tty_cell';
import text_nodes from 'fixtures/text_nodes';
import {
with_text,
@ -16,69 +19,80 @@ import {
window.innerWidth = 3;
window.innerHeight = 4;
let document_builder;
let frame_builder;
function setup() {
document_builder = new DocumentBuilder();
document_builder.tty_width = 3
document_builder.tty_height = 2 + 2
document_builder.char_width = 1
document_builder.char_height = 2
document_builder.getScreenshotWithText();
document_builder.getScreenshotWithoutText();
document_builder.getScaledScreenshot();
frame_builder = new FrameBuilder(undefined, new Dimensions());
frame_builder.graphics_builder.getScreenshotWithText();
frame_builder.graphics_builder.getScreenshotWithoutText();
frame_builder.graphics_builder.getScaledScreenshot();
sandbox.stub(TTYCell.prototype, 'isHighestLayer').returns(true);
}
describe('Text Builder', () => {
beforeEach(() => {
let getPixelsStub = sandbox.stub(DocumentBuilder.prototype, '_getPixelData');
let getPixelsStub = sandbox.stub(GraphicsBuilder.prototype, '_getPixelData');
getPixelsStub.onCall(0).returns(with_text);
getPixelsStub.onCall(1).returns(without_text);
getPixelsStub.onCall(2).returns(scaled);
setup();
document_builder._text_nodes = text_nodes;
frame_builder.text_builder._text_nodes = text_nodes;
});
it('should convert text nodes to a grid', () => {
document_builder._updateState();
document_builder._positionTextNodes();
const grid = document_builder.tty_grid;
expect(grid[0]).to.deep.equal([
't', [255, 255, 255],
[0, 0, 0],
{
"style": {
"textAlign": "left"
}
}, {
"x": 0,
"y": 0
frame_builder.text_builder._updateState();
frame_builder.text_builder._positionTextNodes();
const grid = frame_builder.text_builder.tty_grid.cells;
expect(grid[0]).to.deep.equal({
index: 0,
rune: 't',
fg_colour: [255, 255, 255],
bg_colour: [0, 0, 0],
parent_element: {
style: {}
},
tty_coords: {
x: 0,
y: 0
},
dom_coords: {
x: 0,
y: 0
}
]);
expect(grid[1]).to.deep.equal([
'e', [255, 255, 255],
[111, 111, 111], {
"style": {
"textAlign": "left"
}
}, {
"x": 1,
"y": 0
});
expect(grid[1]).to.deep.equal({
index: 1,
rune: 'e',
fg_colour: [255, 255, 255],
bg_colour: [111, 111, 111],
parent_element: {
style: {}
},
tty_coords: {
x: 1,
y: 0
},
dom_coords: {
x: 1,
y: 0
}
]);
expect(grid[2]).to.deep.equal([
's', [255, 255, 255],
[0, 0, 0], {
"style": {
"textAlign": "left"
}
}, {
"x": 2,
"y": 0
});
expect(grid[2]).to.deep.equal({
index: 2,
rune: 's',
fg_colour: [255, 255, 255],
bg_colour: [0, 0, 0],
parent_element: {
style: {}
},
tty_coords: {
x: 2,
y: 0
},
dom_coords: {
x: 2,
y: 0
}
]);
expect(grid[3]).to.be.undefined;
expect(grid[4]).to.be.undefined;
expect(grid[5]).to.be.undefined;
});
});
});

3
webext/webpack.config.js
View File

@ -20,8 +20,9 @@ module.exports = {
},
plugins: [
new webpack.DefinePlugin({
// TODO: For production use a different webpack.config.js
DEVELOPMENT: JSON.stringify(true),
TEST: JSON.stringify(false),
// TODO: For production use a different webpack.config.js
PRODUCTION: JSON.stringify(false)
}),
new CopyWebpackPlugin([