diff --git a/interfacer/src/browsh/browsh.go b/interfacer/src/browsh/browsh.go index 511a3f8..25a68de 100644 --- a/interfacer/src/browsh/browsh.go +++ b/interfacer/src/browsh/browsh.go @@ -43,10 +43,8 @@ var ( ffCommandCount = 0 isConnectedToWebExtension = false screen tcell.Screen - frame []string uiHeight = 2 - frameWidth int - frameHeight int + frame = Frame{} State map[string]string defaultFFPrefs = map[string]string{ "browser.startup.homepage": "'https://www.google.com'", diff --git a/interfacer/src/browsh/comms.go b/interfacer/src/browsh/comms.go index cb7f4a2..0120292 100644 --- a/interfacer/src/browsh/comms.go +++ b/interfacer/src/browsh/comms.go @@ -38,8 +38,12 @@ func handleWebextensionCommand(message []byte) { parts := strings.Split(string(message), ",") command := parts[0] switch command { - case "/frame": - frame = parseJSONFrame(strings.Join(parts[1:], ",")) + case "/frame_text": + frame.parseJSONFrameText(strings.Join(parts[1:], ",")) + renderUI() + renderFrame() + case "/frame_pixels": + frame.parseJSONFramePixels(strings.Join(parts[1:], ",")) renderUI() renderFrame() case "/state": @@ -56,18 +60,6 @@ func handleWebextensionCommand(message []byte) { } } -// Frames received from the webextension are 1 dimensional arrays of strings. -// They are made up of a repeating pattern of 7 items: -// ["FG RED", "FG GREEN", "FG BLUE", "BG RED", "BG GREEN", "BG BLUE", "CHARACTER" ...] -func parseJSONFrame(jsonString string) []string { - var frame []string - jsonBytes := []byte(jsonString) - if err := json.Unmarshal(jsonBytes, &frame); err != nil { - Shutdown(err) - } - return frame -} - func parseJSONState(jsonString string) { jsonBytes := []byte(jsonString) if err := json.Unmarshal(jsonBytes, &State); err != nil { @@ -83,8 +75,8 @@ func handleStateChange(oldState map[string]string) { } } if (State["frame_width"] != "" && State["frame_height"] != "") { - frameWidth = toInt(State["frame_width"]) - frameHeight = toInt(State["frame_height"]) + frame.width = toInt(State["frame_width"]) + frame.height = toInt(State["frame_height"]) } } diff --git a/interfacer/src/browsh/frame_builder.go b/interfacer/src/browsh/frame_builder.go new file mode 100644 index 0000000..21c2b2b --- /dev/null +++ b/interfacer/src/browsh/frame_builder.go @@ -0,0 +1,168 @@ +package browsh + +import ( + "encoding/json" + "unicode" + "fmt" + + "github.com/gdamore/tcell" +) + +// Frame is a single frame for the entire DOM. The TTY is merely a window onto a +// region of this frame. +type Frame struct { + width int + height int + pixels [][2]tcell.Color + text [][]rune + textColours []tcell.Color + cells []cell +} + +type cell struct { + character []rune + fgColour tcell.Color + bgColour tcell.Color +} + +// Text frames received from the webextension are 1 dimensional arrays of strings. +// They are made up of a repeating pattern of 4 items: +// ["RED", "GREEN", "BLUE", "CHARACTER" ...] +func (f *Frame) parseJSONFrameText(jsonString string) { + if (len(f.pixels) == 0) { f.preFillPixelsWithBlack() } + var index, textIndex int + var frame []string + jsonBytes := []byte(jsonString) + if err := json.Unmarshal(jsonBytes, &frame); err != nil { + Shutdown(err) + } + if (len(frame) < f.width * (f.height / 2 ) * 4) { + Log( + fmt.Sprintf( + "Not parsing small text frame. Data length: %d, current dimensions: %dx(%d/2)*4=%d", + len(frame), + f.width, + f.height, + f.width * (f.height / 2 ) * 4)) + return + } + f.cells = make([]cell, (f.height / 2) * f.width) + f.text = make([][]rune, (f.height / 2) * f.width) + f.textColours = make([]tcell.Color, (f.height / 2) * f.width) + for y := 0; y < f.height / 2; y++ { + for x := 0; x < f.width; x++ { + index = ((f.width * y) + x) + textIndex = index * 4 + f.textColours[index] = tcell.NewRGBColor( + toInt32(frame[textIndex + 0]), + toInt32(frame[textIndex + 1]), + toInt32(frame[textIndex + 2]), + ) + f.text[index] = []rune(frame[textIndex + 3]) + f.buildCell(x, y); + } + } +} + +// This covers the rare situation where a text frame has been sent before any pixel +// data has been populated. +func (f *Frame) preFillPixelsWithBlack() { + f.pixels = make([][2]tcell.Color, f.height * f.width) + for i := range f.pixels { + f.pixels[i] = [2]tcell.Color{ + tcell.NewRGBColor(0, 0, 0), + tcell.NewRGBColor(0, 0, 0), + } + } +} + +// Pixel frames received from the webextension are 1 dimensional arrays of strings. +// They are made up of a repeating pattern of 6 items: +// ["FG RED", "FG GREEN", "FG BLUE", "BG RED", "BG GREEN", "BG BLUE" ...] +// TODO: Can these be sent as binary blobs? +func (f *Frame) parseJSONFramePixels(jsonString string) { + if (len(f.text) == 0) { return } + var index, indexFg, indexBg, pixelIndexFg, pixelIndexBg int + var frame []string + jsonBytes := []byte(jsonString) + if err := json.Unmarshal(jsonBytes, &frame); err != nil { + Shutdown(err) + } + if (len(frame) != f.width * f.height * 3) { + Log( + fmt.Sprintf( + "Not parsing pixels frame. Data length: %d, current dimensions: %dx%d*3=%d", + len(frame), + f.width, + f.height, + f.width * f.height * 3)) + return + } + f.cells = make([]cell, (f.height / 2) * f.width) + f.pixels = make([][2]tcell.Color, f.height * f.width) + for y := 0; y < f.height; y += 2 { + for x := 0; x < f.width; x++ { + index = (f.width * (y / 2)) + x + indexBg = (f.width * y) + x + indexFg = (f.width * (y + 1)) + x + pixelIndexBg = indexBg * 3 + pixelIndexFg = indexFg * 3 + pixels := [2]tcell.Color{ + tcell.NewRGBColor( + toInt32(frame[pixelIndexBg + 0]), + toInt32(frame[pixelIndexBg + 1]), + toInt32(frame[pixelIndexBg + 2]), + ), + tcell.NewRGBColor( + toInt32(frame[pixelIndexFg + 0]), + toInt32(frame[pixelIndexFg + 1]), + toInt32(frame[pixelIndexFg + 2]), + ), + } + f.pixels[index] = pixels + f.buildCell(x, y / 2); + } + } +} + +// This is where we implement the UTF8 half-block trick. +// This a half-block: "▄", notice how it takes up precisely half a text cell. This +// means that we can get 2 pixel colours from it, the top pixel comes from setting +// the background colour and the bottom pixel comes from setting the foreground +// colour, namely the colour of the text. +func (f *Frame) buildCell(x int, y int) { + index := ((f.width * y) + x) + if (index >= len(f.pixels)) { return } // TODO: There must be a better way + character, fgColour := f.getCharacterAt(index) + pixelFg, bgColour := f.getPixelColoursAt(index) + if (isCharacterTransparent(character)) { + character = []rune("▄") + fgColour = pixelFg + } + f.addCell(index, fgColour, bgColour, character) +} + +func (f *Frame) getCharacterAt(index int) ([]rune, tcell.Color) { + character := f.text[index] + colour := f.textColours[index] + return character, colour +} + +func (f *Frame) getPixelColoursAt(index int) (tcell.Color, tcell.Color) { + bgColour := f.pixels[index][0] + fgColour := f.pixels[index][1] + return fgColour, bgColour +} + +func isCharacterTransparent(character []rune) bool { + return string(character) == "" || unicode.IsSpace(character[0]); +} + +func (f *Frame) addCell(index int, fgColour, bgColour tcell.Color, character []rune) { + newCell := cell{ + fgColour: fgColour, + bgColour: bgColour, + character: character, + } + f.cells[index] = newCell +} diff --git a/interfacer/src/browsh/frame_builder_test.go b/interfacer/src/browsh/frame_builder_test.go new file mode 100644 index 0000000..142d120 --- /dev/null +++ b/interfacer/src/browsh/frame_builder_test.go @@ -0,0 +1,66 @@ +package browsh + +import ( + "testing" + "encoding/json" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestFrameBuilder(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Frame builder tests") +} + +var testFrame Frame + +var frameJSONText, _ = json.Marshal([]string{ + "77", "77", "77", "A", + "101", "101", "101", "b", + "102", "102", "102", "c", + "103", "103", "103", "", +}) +var frameText = string(frameJSONText) + +var frameJSONPixels, _ = json.Marshal([]string{ + "254", "254", "254", "111", "111", "111", + "1", "1", "1", "2", "2", "2", + "3", "3", "3", "4", "4", "4", + "123", "123", "123", "200", "200", "200", +}) +var framePixels = string(frameJSONPixels) + + +var _ = Describe("Frame struct", func() { + BeforeEach(func() { + testFrame = Frame{width: 2, height: 4} + testFrame.parseJSONFrameText(frameText) + }) + + It("should parse JSON frame text", func() { + Expect(testFrame.cells[0].character[0]).To(Equal('A')) + Expect(testFrame.cells[1].character[0]).To(Equal('b')) + Expect(testFrame.cells[2].character[0]).To(Equal('c')) + Expect(testFrame.cells[3].character[0]).To(Equal('▄')) + }) + + It("should parse JSON pixels (for text-less cells)", func() { + var r, g, b int32 + testFrame.parseJSONFramePixels(framePixels) + r, g, b = testFrame.cells[3].fgColour.RGB() + Expect([3]int32{r, g, b}).To(Equal([3]int32{200, 200, 200})) + r, g, b = testFrame.cells[3].bgColour.RGB() + Expect([3]int32{r, g, b}).To(Equal([3]int32{4, 4, 4})) + }) + + It("should parse JSON pixels (using text for foreground)", func() { + var r, g, b int32 + testFrame.parseJSONFramePixels(framePixels) + r, g, b = testFrame.cells[0].fgColour.RGB() + Expect([3]int32{r, g, b}).To(Equal([3]int32{77, 77, 77})) + r, g, b = testFrame.cells[0].bgColour.RGB() + Expect([3]int32{r, g, b}).To(Equal([3]int32{254, 254, 254})) + }) +}) + diff --git a/interfacer/src/browsh/tty.go b/interfacer/src/browsh/tty.go index 1ef5a6f..50df834 100644 --- a/interfacer/src/browsh/tty.go +++ b/interfacer/src/browsh/tty.go @@ -95,7 +95,7 @@ func handleScrolling(ev *tcell.EventKey) { } func limitScroll(height int) { - maxYScroll := (frameHeight / 2) - height + maxYScroll := (frame.height / 2) - height if (yScroll > maxYScroll) { yScroll = maxYScroll } if (yScroll < 0) { yScroll = 0 } } @@ -122,23 +122,17 @@ func renderAll() { // Render the tabs and URL bar // TODO: Temporary function, UI rendering should all be moved into this CLI app func renderUI() { + return var styling = tcell.StyleDefault - var character string var runeChars []rune width, _ := screen.Size() index := 0 for y := 0; y < uiHeight ; y++ { for x := 0; x < width; x++ { - styling = styling.Foreground(getRGBColor(index)) - index += 3 - styling = styling.Background(getRGBColor(index)) - index += 3 - character = frame[index] - runeChars = []rune(character) + styling = styling.Foreground(frame.cells[index].fgColour) + styling = styling.Background(frame.cells[index].bgColour) + runeChars = frame.cells[index].character index++ - if (character == "WIDE") { - continue - } screen.SetCell(x, y, styling, runeChars[0]) } } @@ -150,35 +144,23 @@ func renderUI() { // will try to minimise rendering commands by only rendering parts of the terminal // that have changed. func renderFrame() { + if (len(frame.pixels) == 0 || len(frame.text) == 0) { return } var styling = tcell.StyleDefault - var character string var runeChars []rune width, height := screen.Size() - uiSize := uiHeight * width * 7 - if (len(frame) == uiSize) { - Log("Not rendering zero-size frame data") - return - } - if (frameWidth == 0 || frameHeight == 0) { + if (frame.width == 0 || frame.height == 0) { Log("Not rendering frame with a zero dimension") return } index := 0 for y := 0; y < height - uiHeight; y++ { for x := 0; x < width; x++ { - index = ((y + yScroll) * frameWidth * 7) + ((x + xScroll) * 7) - index += uiSize + index = ((y + yScroll) * frame.width) + ((x + xScroll)) if (!checkCell(index, x + xScroll, y + yScroll)) { return } - styling = styling.Foreground(getRGBColor(index)) - index += 3 - styling = styling.Background(getRGBColor(index)) - index += 3 - character = frame[index] - runeChars = []rune(character) - index++ - if (character == "WIDE") { - continue - } + styling = styling.Foreground(frame.cells[index].fgColour) + styling = styling.Background(frame.cells[index].bgColour) + runeChars = frame.cells[index].character + if (len(runeChars) == 0) { continue } // TODO: shouldn't need this screen.SetCell(x, y + uiHeight, styling, runeChars[0]) } } @@ -201,23 +183,12 @@ func overlayPageStatusMessage(height int) { } func checkCell(index, x, y int) bool { - for i := 0; i < 7; i++ { - if (index + i >= len(frame) || frame[index + i] == "") { - message := fmt.Sprintf("Blank frame data (size: %d) at %dx%d, index:%d/%d", len(frame), x, y, index, i) - Log(message) - Log(fmt.Sprintf("%d", yScroll)) - return false; - } + if (index >= len(frame.cells)) { + message := fmt.Sprintf( + "Blank frame data (size: %d) at %dx%d, index: %d", + len(frame.cells), x, y, index) + Log(message) + return false; } return true; } - -// Given a raw frame from the webextension, find the RGB colour at a given -// 1 dimensional index. -func getRGBColor(index int) tcell.Color { - rgb := frame[index:index + 3] - return tcell.NewRGBColor( - toInt32(rgb[0]), - toInt32(rgb[1]), - toInt32(rgb[2])) -} diff --git a/webext/src/background/tab_commands_mixin.js b/webext/src/background/tab_commands_mixin.js index 2f1fef5..5c98190 100644 --- a/webext/src/background/tab_commands_mixin.js +++ b/webext/src/background/tab_commands_mixin.js @@ -10,10 +10,11 @@ export default (MixinBase) => class extends MixinBase { const parts = message.split(','); const command = parts[0]; switch (command) { - case '/frame': - this._current_frame = JSON.parse(message.slice(7)); - this._applyUI(); - this._sendCurrentFrame(); + case '/frame_text': + this.sendToTerminal(`/frame_text,${message.slice(12)}`); + break; + case '/frame_pixels': + this.sendToTerminal(`/frame_pixels,${message.slice(14)}`); break; case '/tab_info': this.currentTab().info = JSON.parse(utils.rebuildArgsToSingleArg(parts)); @@ -27,7 +28,7 @@ export default (MixinBase) => class extends MixinBase { case '/status': if (this._current_frame) { this.updateStatus(parts[1]); - this._sendCurrentFrame(); + this.sendState(); } break; case `/log`: @@ -52,14 +53,6 @@ export default (MixinBase) => class extends MixinBase { this.sendState(); } - _sendCurrentFrame() { - this.sendState(); - // TODO: I struggled with unmarshalling a mixed array in Golang so I'm crudely - // just casting everything to a string for now. - this._current_frame = this._current_frame.map((i) => i.toString()); - this.sendToTerminal(`/frame,${JSON.stringify(this._current_frame)}`); - } - updateStatus(status, message = '') { if (typeof this._current_frame === 'undefined') return; let status_message; diff --git a/webext/src/background/tty_commands_mixin.js b/webext/src/background/tty_commands_mixin.js index 515d065..c821830 100644 --- a/webext/src/background/tty_commands_mixin.js +++ b/webext/src/background/tty_commands_mixin.js @@ -26,7 +26,7 @@ export default (MixinBase) => class extends MixinBase { // TODO: cancel the current FPS iteration when using this _.throttle(() => { this.sendToCurrentTab('/request_frame') - }, 50); + }, 250); break; case '/status': this.updateStatus('', parts.slice(1).join(',')); diff --git a/webext/src/dom/common_mixin.js b/webext/src/dom/common_mixin.js index ea30a51..cb52e9b 100644 --- a/webext/src/dom/common_mixin.js +++ b/webext/src/dom/common_mixin.js @@ -23,7 +23,7 @@ export default (MixinBase) => class extends MixinBase { // If you're logging large objects and using a high-ish FPS (<1000ms) then you might // crash the browser. So use this function instead. firstFrameLog(...logs) { - //if (this._is_first_frame_finished) return; + if (this._is_first_frame_finished) return; if (DEVELOPMENT) { this.log(logs); } diff --git a/webext/src/dom/dimensions.js b/webext/src/dom/dimensions.js index f51aa25..c396315 100644 --- a/webext/src/dom/dimensions.js +++ b/webext/src/dom/dimensions.js @@ -101,9 +101,18 @@ export default class extends utils.mixins(CommonMixin) { // Note that it treats the height of a single TTY cell as containing 2 pixels. Therefore // a TTY of 4x4 will have frame dimensions of 4x8. _updateFrameDimensions() { + let width = this.dom.width * this.scale_factor.width; + let height = this.dom.height * this.scale_factor.height; + width = utils.snap(width); + height = utils.snap(height); + + // Only the height needs to be even because of the UTF8 half-block trick. A single + // TTY cell always contains exactly 2 pseudo pixels. + height = utils.ensureEven(height); + this.frame = { - width: utils.snap(this.dom.width * this.scale_factor.width), - height: utils.snap(this.dom.height * this.scale_factor.height) + width: width, + height: height } } diff --git a/webext/src/dom/frame_builder.js b/webext/src/dom/frame_builder.js deleted file mode 100644 index 07045ca..0000000 --- a/webext/src/dom/frame_builder.js +++ /dev/null @@ -1,167 +0,0 @@ -import utils from 'utils'; - -import CommonMixin from 'dom/common_mixin'; -import GraphicsBuilder from 'dom/graphics_builder'; -import TextBuilder from 'dom/text_builder'; - -// Takes the graphics and text from the current DOM, combines them, then -// sends it to the background process where the rest of the UI, like tabs, -// address bar, etc will be added. -export default class extends utils.mixins(CommonMixin) { - constructor(channel, dimensions) { - super(); - this.channel = channel; - this.is_graphics_mode = true; - this.dimensions = dimensions; - this.dimensions.update(); - this.graphics_builder = new GraphicsBuilder(channel, dimensions); - this.text_builder = new TextBuilder(channel, dimensions, this.graphics_builder); - } - - makeFrame() { - this.dimensions.update(); - this._compileFrame(); - this._buildFrame(); - this._is_first_frame_finished = true; - } - - // The user clicks on a TTY grid which has a significantly lower resolution than the - // actual browser window. So we scale the coordinates up as if the user clicked on the - // the central "pixel" of a TTY cell. - // - // Furthermore if the TTY click is on a readable character then the click is proxied - // to the original position of the character before TextBuilder snapped the character into - // position. - getDOMCoordsFromMouseCoords(x, y) { - let dom_x, dom_y, char, original_position; - const index = (y * this.dimensions.frame.width) + x; - if (this.text_builder.tty_grid.cells[index] !== undefined) { - char = this.text_builder.tty_grid.cells[index].rune; - } else { - char = false; - } - if (!char || char === '▄') { - dom_x = (x * this.dimensions.char.width); - dom_y = (y * this.dimensions.char.height); - } else { - // Recall that text can be shifted from its original position in the browser in order - // to snap it consistently to the TTY grid. - original_position = this.text_builder.tty_grid.cells[index].dom_coords; - dom_x = original_position.x; - dom_y = original_position.y; - } - return [ - dom_x + (this.dimensions.char.width / 2), - dom_y + (this.dimensions.char.height / 2) - ]; - } - - _compileFrame() { - if (!this._is_first_frame_finished || this.dimensions.dom.is_new) { - this.buildText(); - } - this.graphics_builder.getScaledScreenshot(); - } - - // All the processes necessary to build the text for a frame. It can be CPU intensive - // so we don't want to do it all the time. - buildText() { - this.text_builder.fixJustifiedText(); - this.graphics_builder.getScreenshotWithText(); - this.graphics_builder.getScreenshotWithoutText(); - this.text_builder.buildFormattedText(); - } - - _buildFrame() { - this.logPerformance(() => { - this.__buildFrame(); - }, 'build frame'); - } - - __buildFrame() { - this.frame = []; - this._bg_row = []; - this._fg_row = []; - for (let y = 0; y < this.dimensions.frame.height; y++) { - for (let x = 0; x < this.dimensions.frame.width; x++) { - this._buildPixel(x, y); - } - } - } - - // Note how we have to keep track of 2 rows of pixels in order to create 1 row of - // the terminal. - _buildPixel(x, y) { - const colour = this.graphics_builder.getScaledPixelAt(x, y); - if (this._bg_row.length < this.dimensions.frame.width) { - this._bg_row.push(colour); - } else { - this._fg_row.push(colour); - } - if (this._fg_row.length === this.dimensions.frame.width) { - this._buildTTYRow(parseInt(y / 2)); - this.frame = this.frame.concat(this._row); - this._bg_row = []; - this._fg_row = []; - } - } - - // This is where we implement the UTF8 half-block trick. - // This is a half-black: "▄", notice how it takes up precisely half a text cell. This - // means that we can get 2 pixel colours from it, the top pixel comes from setting - // the background colour and the bottom pixel comes from setting the foreground - // colour, namely the colour of the text. - // - // However we can't just write these "pixels" to a TTY screen, we must collate 2 rows - // of "pixels" for every row of the terminal and then we can render them together. - _buildTTYRow(row_number) { - this._row = []; - const width = this.dimensions.frame.width - for (this._current_col = 0; this._current_col < width; this._current_col++) { - this._cell = this.text_builder.tty_grid.getCellAt(this._current_col, row_number); - if (this._cell === undefined || this._cell.isTransparent()) { - this._addGraphicsBlock(); - } else { - this._handleCharacter(); - } - } - } - - _handleCharacter() { - const padding_count = this._cell.calculateCharWidthPadding(); - if (padding_count > 0) { - for (let p = 0; p < padding_count; p++) { - this._current_column++; - this._addCharacter(' '); - } - } else { - this._addCharacter(); - } - } - - _addCharacter(override_rune = false) { - let char; - if (override_rune) { - char = override_rune - } else { - char = this._cell.rune - } - if (this.is_graphics_mode) { - this._row = this._row.concat( - utils.ttyCell(this._cell.fg_colour, this._cell.bg_colour, char) - ); - } else { - // TODO: Somehow communicate clickable text - this._row = this._row.concat(utils.ttyPlainCell(char)); - } - } - - _addGraphicsBlock() { - const x = this._current_col; - if (this.is_graphics_mode) { - this._row = this._row.concat(utils.ttyCell(this._fg_row[x], this._bg_row[x], '▄')); - } else { - this._row = this._row.concat(utils.ttyPlainCell(' ')); - } - } -} diff --git a/webext/src/dom/graphics_builder.js b/webext/src/dom/graphics_builder.js index 17bd71c..7a937a5 100644 --- a/webext/src/dom/graphics_builder.js +++ b/webext/src/dom/graphics_builder.js @@ -15,6 +15,16 @@ export default class extends utils.mixins(CommonMixin) { this._ctx = this._off_screen_canvas.getContext('2d'); } + sendFrame() { + this.getScaledScreenshot(); + this._serialiseFrame(); + if (this.frame.length > 0) { + this.sendMessage(`/frame_pixels,${JSON.stringify(this.frame)}`); + } else { + this.log("Not sending empty pixels frame"); + } + } + // With full-block single-glyph font on getUnscaledFGPixelAt(x, y) { const pixel_data_start = parseInt( @@ -113,7 +123,10 @@ export default class extends utils.mixins(CommonMixin) { this._is_scaled = true; this._hideText(); this._ctx.save(); - this._ctx.scale(this.dimensions.scale_factor.width, this.dimensions.scale_factor.height); + this._ctx.scale( + this.dimensions.scale_factor.width, + this.dimensions.scale_factor.height + ); } _unScaleCanvas() { @@ -149,4 +162,15 @@ export default class extends utils.mixins(CommonMixin) { ); return this._ctx.getImageData(0, 0, width, height).data; } + + _serialiseFrame() { + this.frame = []; + const height = this.dimensions.frame.height; + const width = this.dimensions.frame.width; + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + this.getScaledPixelAt(x, y).map((c) => this.frame.push(c.toString())); + } + } + } } diff --git a/webext/src/dom/manager.js b/webext/src/dom/manager.js index 15cdaf3..9332048 100644 --- a/webext/src/dom/manager.js +++ b/webext/src/dom/manager.js @@ -2,7 +2,8 @@ import utils from 'utils'; import CommonMixin from 'dom/common_mixin'; import Dimensions from 'dom/dimensions'; -import FrameBuilder from 'dom/frame_builder'; +import GraphicsBuilder from 'dom/graphics_builder'; +import TextBuilder from 'dom/text_builder'; // Entrypoint for managing a single tab export default class extends utils.mixins(CommonMixin) { @@ -12,20 +13,36 @@ export default class extends utils.mixins(CommonMixin) { this._setupInit(); } + _postCommsConstructor() { + this.dimensions.channel = this.channel; + this.dimensions.update(); + this.graphics_builder = new GraphicsBuilder(this.channel, this.dimensions); + this.text_builder = new TextBuilder(this.channel, this.dimensions, this.graphics_builder); + this.text_builder.sendFrame(); + } + sendFrame() { - this.frame_builder.makeFrame(); + this.dimensions.update() + if (this.dimensions.dom.is_new) { + this.text_builder.sendFrame(); + } + this.graphics_builder.sendFrame(); this._sendTabInfo(); if (!this._is_first_frame_finished) { this.sendMessage('/status,parsing_complete'); } - if (this.frame_builder.frame.length > 0) { - this.sendMessage(`/frame,${JSON.stringify(this.frame_builder.frame)}`); - } else { - this.log("Not sending empty frame"); - } this._is_first_frame_finished = true; } + _postCommsInit() { + this.log('Webextension postCommsInit()'); + this._postCommsConstructor(); + this._sendTabInfo(); + this.sendMessage('/status,page_init'); + this._listenForBackgroundMessages(); + this._startWindowEventListeners() + } + _setupInit() { // TODO: Can we not just boot up as soon as we detect the background script? document.addEventListener("DOMContentLoaded", () => { @@ -67,16 +84,6 @@ export default class extends utils.mixins(CommonMixin) { this.log(error); } - _postCommsInit() { - this.log('Webextension postCommsInit()'); - this.dimensions.channel = this.channel; - this.frame_builder = new FrameBuilder(this.channel, this.dimensions); - this._sendTabInfo(); - this.sendMessage('/status,page_init'); - this._listenForBackgroundMessages(); - this._startWindowEventListeners() - } - _startWindowEventListeners() { window.addEventListener("unload", () => { this.sendMessage('/status,window_unload') @@ -185,7 +192,7 @@ export default class extends utils.mixins(CommonMixin) { } _mouseAction(type, x, y) { - const [dom_x, dom_y] = this.frame_builder.getDOMCoordsFromMouseCoords(x, y); + const [dom_x, dom_y] = this._getDOMCoordsFromMouseCoords(x, y); const element = document.elementFromPoint( dom_x - window.scrollX, dom_y - window.scrollY @@ -199,6 +206,37 @@ export default class extends utils.mixins(CommonMixin) { element.dispatchEvent(event); } + // The user clicks on a TTY grid which has a significantly lower resolution than the + // actual browser window. So we scale the coordinates up as if the user clicked on the + // the central "pixel" of a TTY cell. + // + // Furthermore if the TTY click is on a readable character then the click is proxied + // to the original position of the character before TextBuilder snapped the character into + // position. + _getDOMCoordsFromMouseCoords(x, y) { + let dom_x, dom_y, char, original_position; + const index = (y * this.dimensions.frame.width) + x; + if (this.text_builder.tty_grid.cells[index] !== undefined) { + char = this.text_builder.tty_grid.cells[index].rune; + } else { + char = false; + } + if (!char || char === '▄') { + dom_x = (x * this.dimensions.char.width); + dom_y = (y * this.dimensions.char.height); + } else { + // Recall that text can be shifted from its original position in the browser in order + // to snap it consistently to the TTY grid. + original_position = this.text_builder.tty_grid.cells[index].dom_coords; + dom_x = original_position.x; + dom_y = original_position.y; + } + return [ + dom_x + (this.dimensions.char.width / 2), + dom_y + (this.dimensions.char.height / 2) + ]; + } + _sendTabInfo() { const title_object = document.getElementsByTagName("title"); let info = { diff --git a/webext/src/dom/text_builder.js b/webext/src/dom/text_builder.js index 062072a..d1b47ee 100644 --- a/webext/src/dom/text_builder.js +++ b/webext/src/dom/text_builder.js @@ -12,7 +12,9 @@ export default class extends utils.mixins(CommonMixin) { super(); this.channel = channel; this.dimensions = dimensions; + this.graphics_builder = graphics_builder; this.tty_grid = new TTYGrid(dimensions, graphics_builder); + this.frame = []; this._parse_started_elements = []; // A `range` is the DOM's representation of elements and nodes as they are rendered in // the DOM. Think of the 'range' that is created when you select/highlight text for @@ -20,8 +22,20 @@ export default class extends utils.mixins(CommonMixin) { this._range = document.createRange(); } + sendFrame() { + this.buildFormattedText(); + this._serialiseFrame(); + if (this.frame.length > 0) { + this.sendMessage(`/frame_text,${JSON.stringify(this.frame)}`); + } else { + this.log("Not sending empty text frame"); + } + } + buildFormattedText() { this._updateState(); + this.graphics_builder.getScreenshotWithText(); + this.graphics_builder.getScreenshotWithoutText(); this._getTextNodes(); this._positionTextNodes(); } @@ -45,6 +59,12 @@ export default class extends utils.mixins(CommonMixin) { }, 'position text nodes'); } + _serialiseFrame() { + this.logPerformance(() => { + this.__serialiseFrame(); + }, 'serialise text frame'); + } + // Search through every node in the DOM looking for displayable text. __getTextNodes() { this._text_nodes = []; @@ -77,7 +97,8 @@ export default class extends utils.mixins(CommonMixin) { } _formatText() { - this._normaliseWhitespace() + this._normaliseWhitespace(); + this._fixJustifiedText(); } // Justified text uses the space between words to stretch a line to perfectly fit from @@ -93,11 +114,8 @@ export default class extends utils.mixins(CommonMixin) { // even by a find-replace on all occurrences of 'justify'? // * Yet another thing, the style change doesn't actually get picked up until the // next frame. Thus why the loop is independent of the `positionTextNodes()` loop. - fixJustifiedText() { - this._getTextNodes(); - for (const node of this._text_nodes) { - node.parentElement.style.textAlign = 'left'; - } + _fixJustifiedText() { + this._node.parentElement.style.textAlign = 'left'; } // The need for this wasn't immediately obvious to me. The fact is that the DOM stores @@ -277,6 +295,28 @@ export default class extends utils.mixins(CommonMixin) { } } + __serialiseFrame() { + let cell, index; + this.frame = []; + const height = this.dimensions.frame.height / 2; + const width = this.dimensions.frame.width; + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + index = (y * width) + x; + cell = this.tty_grid.cells[index]; + if (cell === undefined) { + this.frame.push("0") + this.frame.push("0") + this.frame.push("0") + this.frame.push("") + } else { + cell.fg_colour.map((c) => this.frame.push(c.toString())); + this.frame.push(cell.rune); + } + } + } + } + // Purely for debugging. // // Draws a red border around all the DOMClientRect nodes. diff --git a/webext/src/dom/tty_cell.js b/webext/src/dom/tty_cell.js index ba152fb..b1bb008 100644 --- a/webext/src/dom/tty_cell.js +++ b/webext/src/dom/tty_cell.js @@ -1,5 +1,3 @@ -import charWidthInTTY from 'string-width'; - // A single cell on the TTY grid export default class { // When a character clobbers another character in the grid, we can't use our @@ -28,25 +26,5 @@ export default class { } return element.browsh_calculated_styles; } - - isTransparent() { - const is_undefined = this.rune === undefined; - const is_empty = this.rune === ''; - const is_space = /^\s+$/.test(this.rune); - const is_not_worth_printing = is_empty || is_space || is_undefined; - return is_not_worth_printing; - } - - // Deal with UTF8 characters that take up more than a single cell in the TTY. - // Eg; 比如说 - // TODO: - // 1. Do all terminals deal with wide characters the same? - // 2. Use CSS or JS so that wide characters actually flow in the DOM as 2 - // monospaced characters. This will allow pages of nothing but wide - // characters to render/flow as closely as possible ot how they will appear - // in the TTY. - calculateCharWidthPadding() { - return charWidthInTTY(this.rune) - 1; - } } diff --git a/webext/src/utils.js b/webext/src/utils.js index d1ef3c0..f948fa9 100644 --- a/webext/src/utils.js +++ b/webext/src/utils.js @@ -21,6 +21,11 @@ export default { return parseInt(Math.round(number)); }, + ensureEven: function (number) { + if (number % 2) { number++ } + return number; + }, + rebuildArgsToSingleArg: function (args) { return args.slice(1).join(','); } diff --git a/webext/test/fixtures/canvas_pixels.js b/webext/test/fixtures/canvas_pixels.js index 9aca5fd..45cb828 100644 --- a/webext/test/fixtures/canvas_pixels.js +++ b/webext/test/fixtures/canvas_pixels.js @@ -1,8 +1,8 @@ export let with_text = [ 255,255,255,0, 255,255,255,0, 255,255,255,0, 255,255,255,0, 255,255,255,0, 255,255,255,0, - 111,111,111,0, 0 ,0 ,0 ,0, 111,111,111,0, - 0 ,0 ,0 ,0, 111,111,111,0, 0 ,0 , 0,0 + 255,255,255,0, 0 ,0 ,0 ,0, 111,111,111,0, + 255,255,255,0, 111,111,111,0, 0 ,0 , 0,0 ]; export let without_text = [ diff --git a/webext/test/fixtures/cells.js b/webext/test/fixtures/cells.js deleted file mode 100644 index e7ff97a..0000000 --- a/webext/test/fixtures/cells.js +++ /dev/null @@ -1,25 +0,0 @@ -import TTYCell from "dom/tty_cell"; - -let base = build( - [ - '', '😐', '', - '😄', '', '😂' - ], - [111, 111, 111], - [222, 222, 222] -); - -export default base; - -function build(text, fg_colour, bg_colour) { - let cell; - let grid = []; - for(const character of text) { - cell = new TTYCell(); - cell.rune = character; - cell.fg_colour = fg_colour; - cell.bg_colour = bg_colour; - grid.push(cell); - } - return grid; -} diff --git a/webext/test/fixtures/text_nodes.js b/webext/test/fixtures/text_nodes.js index b835ef4..a9f3e3e 100644 --- a/webext/test/fixtures/text_nodes.js +++ b/webext/test/fixtures/text_nodes.js @@ -1,4 +1,4 @@ -// It'd be nice to somehow automate the creation of the coordinates here +// TODO: It'd be nice to somehow automate the creation of the coordinates here let base = { textContent: "\n testing nodes", @@ -7,7 +7,7 @@ let base = { }, bounding_box: { top: 0.1, - bottom: 4.1, + bottom: 2.1, left: 0.1, right: 7.1, width: 7.1 @@ -20,7 +20,7 @@ let base = { }, { // 'nodes' - top: 4.1, + top: 2.1, left: 0.1, width: 5.1 }] diff --git a/webext/test/frame_builder_spec.js b/webext/test/frame_builder_spec.js deleted file mode 100644 index 3efef86..0000000 --- a/webext/test/frame_builder_spec.js +++ /dev/null @@ -1,35 +0,0 @@ -import sandbox from 'helper'; -import {expect} from 'chai'; - -import Dimensions from 'dom/dimensions'; -import FrameBuilder from 'dom/frame_builder'; -import GraphicsBuilder from 'dom/graphics_builder'; -import TextBuilder from 'dom/text_builder'; -import canvas_pixels from 'fixtures/canvas_pixels'; -import cells from 'fixtures/cells'; - -describe('Frame Builder', ()=> { - let frame_builder; - - beforeEach(()=> { - global.document.documentElement.scrollHeight = 4; - frame_builder = new FrameBuilder(undefined, new Dimensions()); - frame_builder.text_builder.tty_grid.cells = cells; - sandbox.stub(GraphicsBuilder.prototype, '_getPixelData').returns(canvas_pixels); - sandbox.stub(TextBuilder.prototype, 'buildFormattedText').returns(); - sandbox.stub(TextBuilder.prototype, 'fixJustifiedText').returns(); - }); - - it('should merge pixels and text into a 1D array', () => { - frame_builder.makeFrame(); - const frame = frame_builder.frame.join(','); - expect(frame).to.eq( - '0,0,0,111,111,111,▄,' + - '111,111,111,222,222,222,😐,' + - '0,0,0,111,111,111,▄,' + - '111,111,111,222,222,222,😄,' + - '111,111,111,0,0,0,▄,' + - '111,111,111,222,222,222,😂' - ); - }); -}); diff --git a/webext/test/graphics_builder_spec.js b/webext/test/graphics_builder_spec.js new file mode 100644 index 0000000..7efb469 --- /dev/null +++ b/webext/test/graphics_builder_spec.js @@ -0,0 +1,31 @@ +import sandbox from 'helper'; +import { expect } from 'chai'; + +import Dimensions from 'dom/dimensions'; +import GraphicsBuilder from 'dom/graphics_builder'; +import { scaled } from 'fixtures/canvas_pixels'; + +let graphics_builder; + +function setup() { + let dimensions = new Dimensions(); + graphics_builder = new GraphicsBuilder(undefined, dimensions); + graphics_builder.getScaledScreenshot(); +} + +describe('Graphics Builder', () => { + beforeEach(() => { + let getPixelsStub = sandbox.stub(GraphicsBuilder.prototype, '_getPixelData'); + getPixelsStub.onCall(0).returns(scaled); + setup(); + }); + + it('should serialise a scaled frame', () => { + graphics_builder._serialiseFrame(); + expect(graphics_builder.frame.length).to.equal(36); + expect(graphics_builder.frame[0]).to.equal('111'); + expect(graphics_builder.frame[3]).to.equal('0'); + expect(graphics_builder.frame[32]).to.equal('111'); + expect(graphics_builder.frame[35]).to.equal('0'); + }); +}); diff --git a/webext/test/helper.js b/webext/test/helper.js index a709209..1f14da3 100644 --- a/webext/test/helper.js +++ b/webext/test/helper.js @@ -1,7 +1,6 @@ import sinon from 'sinon'; import Dimensions from 'dom/dimensions'; -import FrameBuilder from 'dom/frame_builder'; import GraphicsBuilder from 'dom/graphics_builder'; import MockRange from 'mocks/range' @@ -9,11 +8,11 @@ var sandbox = sinon.sandbox.create(); beforeEach(() => { sandbox.stub(Dimensions.prototype, '_getOrCreateMeasuringBox').returns(element); + sandbox.stub(Dimensions.prototype, 'sendMessage').returns(true); sandbox.stub(GraphicsBuilder.prototype, '_hideText').returns(true); sandbox.stub(GraphicsBuilder.prototype, '_showText').returns(true); sandbox.stub(GraphicsBuilder.prototype, '_scaleCanvas').returns(true); sandbox.stub(GraphicsBuilder.prototype, '_unScaleCanvas').returns(true); - sandbox.stub(FrameBuilder.prototype, 'sendMessage').returns(true); }); afterEach(() => { @@ -45,13 +44,19 @@ global.document = { }, documentElement: { scrollWidth: 3, - scrollHeight: 8 + scrollHeight: 4 }, location: { href: 'https://www.google.com' }, scrollX: 0, - scrollY: 0 + scrollY: 0, + + // To save us hand-writing large pixel arrays, let's just have an unrealistically + // small window, it's not a problem, because we'll never actually have to view real + // webpages on it. + innerWidth: 3, + innerHeight: 4 }; global.DEVELOPMENT = false; diff --git a/webext/test/text_builder_spec.js b/webext/test/text_builder_spec.js index 6ee33a7..aefce72 100644 --- a/webext/test/text_builder_spec.js +++ b/webext/test/text_builder_spec.js @@ -2,8 +2,8 @@ import sandbox from 'helper'; import { expect } from 'chai'; import Dimensions from 'dom/dimensions'; -import FrameBuilder from 'dom/frame_builder'; import GraphicsBuilder from 'dom/graphics_builder'; +import TextBuilder from 'dom/text_builder'; import TTYCell from 'dom/tty_cell'; import text_nodes from 'fixtures/text_nodes'; import { @@ -12,20 +12,15 @@ import { scaled } from 'fixtures/canvas_pixels'; - -// To save us hand-writing large pixel arrays, let's just have an unrealistically -// small window, it's not a problem, because we'll never actually have to view real -// webpages on it. -window.innerWidth = 3; -window.innerHeight = 4; - -let frame_builder; +let graphics_builder, text_builder; function setup() { - frame_builder = new FrameBuilder(undefined, new Dimensions()); - frame_builder.graphics_builder.getScreenshotWithText(); - frame_builder.graphics_builder.getScreenshotWithoutText(); - frame_builder.graphics_builder.getScaledScreenshot(); + let dimensions = new Dimensions(); + graphics_builder = new GraphicsBuilder(undefined, dimensions); + text_builder = new TextBuilder(undefined, dimensions, graphics_builder); + graphics_builder.getScreenshotWithText(); + graphics_builder.getScreenshotWithoutText(); + graphics_builder.getScaledScreenshot(); sandbox.stub(TTYCell.prototype, 'isHighestLayer').returns(true); } @@ -36,20 +31,22 @@ describe('Text Builder', () => { getPixelsStub.onCall(1).returns(without_text); getPixelsStub.onCall(2).returns(scaled); setup(); - frame_builder.text_builder._text_nodes = text_nodes; + text_builder._text_nodes = text_nodes; + text_builder._updateState(); + text_builder._positionTextNodes(); }); - it('should convert text nodes to a grid', () => { - frame_builder.text_builder._updateState(); - frame_builder.text_builder._positionTextNodes(); - const grid = frame_builder.text_builder.tty_grid.cells; + it('should convert text nodes to a grid of cell objects', () => { + const grid = text_builder.tty_grid.cells; expect(grid[0]).to.deep.equal({ index: 0, rune: 't', fg_colour: [255, 255, 255], bg_colour: [0, 0, 0], parent_element: { - style: {} + style: { + textAlign: "left" + } }, tty_coords: { x: 0, @@ -60,39 +57,23 @@ describe('Text Builder', () => { y: 0 } }); - expect(grid[1]).to.deep.equal({ - index: 1, - rune: 'e', - fg_colour: [255, 255, 255], - bg_colour: [111, 111, 111], - parent_element: { - style: {} - }, - tty_coords: { - x: 1, - y: 0 - }, - dom_coords: { - x: 1, - y: 0 - } - }); - expect(grid[2]).to.deep.equal({ - index: 2, - rune: 's', - fg_colour: [255, 255, 255], - bg_colour: [0, 0, 0], - parent_element: { - style: {} - }, - tty_coords: { - x: 2, - y: 0 - }, - dom_coords: { - x: 2, - y: 0 - } - }); + expect(grid[5]).to.equal(undefined); + }); + + it('should ignore spaces on new lines', () => { + const grid = text_builder.tty_grid.cells; + expect(grid[3].rune).to.equal('n'); + expect(grid[3].tty_coords.y).to.equal(1); + }); + + it('should serialise a frame', () => { + text_builder._serialiseFrame(); + expect(text_builder.frame).to.deep.equal([ + '255', '255', '255', 't', + '255', '255', '255', 'e', + '255', '255', '255', 's', + '255', '255', '255', 'n', + '0', '0', '0', '', '0', '0', '0', '' + ]); }); });