TextEdit & Highlighting: performance improvements & changes to architecture

I ran pprof to find what was causing stuttering, and found it to be the
getTabCountInLineAtCol function in TextEdit, because it was iterating
many bytes of the buffer, for each rune rendered. Replaced it with a more
optimal system. Also changed the architecture of the highlighting system
to use a single RegexpRange structure for all regular expressions. This
allows for optimizations and multiline matches in the future.

.gitignore

@@ -3,3 +3,5 @@ qedit*
 
 # Unblock screenshots
 !screenshots/*
+
+*.prof

main.go

@@ -2,15 +2,24 @@ package main
 
 import (
 	"errors"
+	"flag"
 	"fmt"
+	"log"
 	"io/fs"
 	"io/ioutil"
 	"os"
+	"runtime"
+	"runtime/pprof"
 
 	"github.com/fivemoreminix/qedit/ui"
 	"github.com/gdamore/tcell/v2"
 )
 
+var (
+	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to `file`")
+	memprofile = flag.String("memprofile", "", "write memory profile to `file`")
+)
+
 var theme = ui.Theme{
 	"StatusBar": tcell.Style{}.Foreground(tcell.ColorBlack).Background(tcell.ColorSilver),
 }
@@ -32,6 +41,19 @@ func changeFocus(to ui.Component) {
 }
 
 func main() {
+	flag.Parse()
+	if *cpuprofile != "" {
+		f, err := os.Create(*cpuprofile)
+		if err != nil {
+			log.Fatal("Could not create CPU profile: ", err)
+		}
+		defer f.Close()
+		if err := pprof.StartCPUProfile(f); err != nil {
+			log.Fatal("Could not start CPU profile: ", err)
+		}
+		defer pprof.StopCPUProfile()
+	}
+
 	s, e := tcell.NewScreen()
 	if e != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", e)
@@ -43,6 +65,7 @@ func main() {
 	}
 	defer s.Fini() // Useful for handling panics
 
+	var closing bool
 	sizex, sizey := s.Size()
 
 	tabContainer = ui.NewTabContainer(&theme)
@@ -57,6 +80,11 @@ func main() {
 	// Open files from command-line arguments
 	if len(os.Args) > 1 {
 		for i := 1; i < len(os.Args); i++ {
+			if os.Args[i] == "-cpuprofile" || os.Args[i] == "-memprofile" {
+				i++
+				continue
+			}
+
 			_, err := os.Stat(os.Args[i])
 
 			var dirty bool
@@ -167,8 +195,7 @@ func main() {
 		if tabContainer.GetTabCount() > 0 {
 			tabContainer.RemoveTab(tabContainer.GetSelectedTabIdx())
 		} else { // No tabs open; close the editor
-			s.Fini()
+			closing = true
-			os.Exit(0)
 		}
 	}}})
 
@@ -254,7 +281,7 @@ func main() {
 
 	changeFocus(tabContainer) // TabContainer is focused by default
 
-	for {
+	for !closing {
 		s.Clear()
 
 		// Draw background (grey and black checkerboard)
@@ -332,4 +359,16 @@ func main() {
 			focusedComponent.HandleEvent(ev)
 		}
 	}
+
+	if *memprofile != "" {
+		f, err := os.Create(*memprofile)
+		if err != nil {
+			log.Fatal("Could not create memory profile: ", err)
+		}
+		defer f.Close()
+		runtime.GC() // Get updated statistics
+		if err := pprof.WriteHeapProfile(f); err != nil {
+			log.Fatal("Could not write memory profile: ", err)
+		}
+	}
 }

ui/buffer/highlighter.go

@@ -1,6 +1,7 @@
 package buffer
 
 import (
+	"regexp"
 	"sort"
 
 	"github.com/gdamore/tcell/v2"
@@ -25,15 +26,23 @@ func (c *Colorscheme) GetStyle(s Syntax) tcell.Style {
 	return tcell.StyleDefault; // No value for Default; use default style.
 }
 
-type SyntaxData struct {
+type RegexpRegion struct {
+	Start    *regexp.Regexp
+	End      *regexp.Regexp // Should be "$" by default
+	Skip     *regexp.Regexp // Optional
+	Error    *regexp.Regexp // Optional
+	Specials []*regexp.Regexp // Optional (nil or zero len)
+}
+
+type Match struct {
 	Col     int
-	EndLine int
+	EndLine int // Inclusive
-	EndCol  int
+	EndCol  int // Inclusive
 	Syntax  Syntax
 }
 
-// ByCol implements sort.Interface for []SyntaxData based on the Col field.
+// ByCol implements sort.Interface for []Match based on the Col field.
-type ByCol []SyntaxData
+type ByCol []Match
 
 func (c ByCol) Len() int           { return len(c) }
 func (c ByCol) Swap(i, j int)      { c[i], c[j] = c[j], c[i] }
@@ -46,7 +55,7 @@ type Highlighter struct {
 	Language    *Language
 	Colorscheme *Colorscheme
 
-	lineData [][]SyntaxData
+	lineMatches [][]Match
 }
 
 func NewHighlighter(buffer Buffer, lang *Language, colorscheme *Colorscheme) *Highlighter {
@@ -54,50 +63,108 @@ func NewHighlighter(buffer Buffer, lang *Language, colorscheme *Colorscheme) *Hi
 		buffer,
 		lang,
 		colorscheme,
-		make([][]SyntaxData, buffer.Lines()),
+		make([][]Match, buffer.Lines()),
 	}
 }
 
-func (h *Highlighter) Update() {
+// UpdateLines forces the highlighting matches for lines between startLine to
-	if lines := h.Buffer.Lines(); len(h.lineData) < lines {
+// endLine, inclusively, to be updated. It is more efficient to mark lines as
-		h.lineData = append(h.lineData, make([][]SyntaxData, lines)...) // Extend
+// invalidated when changes occur and call UpdateInvalidatedLines(...).
+func (h *Highlighter) UpdateLines(startLine, endLine int) {
+	if lines := h.Buffer.Lines(); len(h.lineMatches) < lines {
+		h.lineMatches = append(h.lineMatches, make([][]Match, lines)...) // Extend
 	}
-	for i := range h.lineData { // Invalidate all line data
+	for i := startLine; i <= endLine && i < len(h.lineMatches); i++ {
-		h.lineData[i] = nil
+		if h.lineMatches[i] != nil {
+			h.lineMatches[i] = h.lineMatches[i][:0] // Shrink slice to zero (hopefully save allocs)
+		}
 	}
 
-	// For each compiled syntax regex:
+	// If the rule k does not have an End, then it can be optimized that we search from the start
-	//   Use FindAllIndex to get all instances of a single match, then for each match found:
+	// of view until the end of view. For any k that has an End, we search for starts from start
-	//     use Find to get the bytes of the match and get the length. Calculate to what line
+	// of buffer, until end of view.
-	//     and column the bytes span and its syntax. Append a SyntaxData to the output.
 
-	bytes := (h.Buffer).Bytes() // Allocates size of the buffer	
+	endLine, endCol := h.Buffer.ClampLineCol(endLine, (h.Buffer).RunesInLineWithDelim(endLine)-1)
+	bytes := (h.Buffer).Slice(0, 0, endLine, endCol) // Allocates size of the buffer	
 
 	for k, v := range h.Language.Rules {
-		indexes := k.FindAllIndex(bytes, -1)
+		indexes := k.Start.FindAllIndex(bytes, -1) // Attempt to find the start match
 		if indexes != nil {
 			for i := range indexes {
+//				if k.End != nil && k.End.String() != "$" { // If this match has a defined end...
+//				}
 				endPos := indexes[i][1] - 1
 				startLine, startCol := h.Buffer.PosToLineCol(indexes[i][0])
 				endLine, endCol := h.Buffer.PosToLineCol(endPos)
 
-				syntaxData := SyntaxData { startCol, endLine, endCol, v }
+				match := Match { startCol, endLine, endCol, v }
 
-				h.lineData[startLine] = append(h.lineData[startLine], syntaxData) // Not sorted
+				h.lineMatches[startLine] = append(h.lineMatches[startLine], match) // Unsorted
 			}
 		}
 	}
+
+	h.validateLines(startLine, endLine) // Marks any "unvalidated" or nil lines as valued
+}
+
+// UpdateInvalidatedLines only updates the highlighting for lines that are invalidated
+// between lines startLine and endLine, inclusively.
+func (h *Highlighter) UpdateInvalidatedLines(startLine, endLine int) {
+	// Move startLine to first line with invalidated changes
+	for startLine <= endLine && startLine < len(h.lineMatches)-1 {
+		if h.lineMatches[startLine] == nil {
+			break
+		}
+		startLine++
+	}
+
+	// Move endLine back to first line at or before endLine with invalidated changes
+	for endLine >= startLine && endLine > 0 {
+		if h.lineMatches[endLine] == nil {
+			break
+		}
+		endLine--
+	}
+
+	if startLine > endLine {
+		return // Do nothing; no invalidated lines
+	}
+
+	h.UpdateLines(startLine, endLine)
+}
+
+func (h *Highlighter) HasInvalidatedLines(startLine, endLine int) bool {
+	for i := startLine; i <= endLine && i < len(h.lineMatches); i++ {
+		if h.lineMatches[i] == nil {
+			return true
+		}
+	}
+	return false
+}
+
+func (h *Highlighter) validateLines(startLine, endLine int) {
+	for i := startLine; i <= endLine && i < len(h.lineMatches); i++ {
+		if h.lineMatches[i] == nil {
+			h.lineMatches[i] = make([]Match, 0)
+		}
+	}
 }
 
-func (h *Highlighter) GetLine(line int) []SyntaxData {
+func (h *Highlighter) InvalidateLines(startLine, endLine int) {
-	if line < 0 || line >= len(h.lineData) {
+	for i := startLine; i <= endLine && i < len(h.lineMatches); i++ {
+		h.lineMatches[i] = nil
+	}
+}
+
+func (h *Highlighter) GetLineMatches(line int) []Match {
+	if line < 0 || line >= len(h.lineMatches) {
 		return nil
 	}
-	data := h.lineData[line]
+	data := h.lineMatches[line]
 	sort.Sort(ByCol(data))
 	return data
 }
 
-func (h *Highlighter) GetStyle(syn SyntaxData) tcell.Style {
+func (h *Highlighter) GetStyle(match Match) tcell.Style {
-	return h.Colorscheme.GetStyle(syn.Syntax)
+	return h.Colorscheme.GetStyle(match.Syntax)
 }

ui/buffer/language.go

@@ -1,7 +1,5 @@
 package buffer
 
-import "regexp"
-
 type Syntax uint8
 
 const (
@@ -19,6 +17,6 @@ const (
 type Language struct {
 	Name      string
 	Filetypes []string // .go, .c, etc.
-	Rules     map[*regexp.Regexp]Syntax
+	Rules     map[*RegexpRegion]Syntax
 	// TODO: add other language details
 }

ui/buffer/rope.go

@@ -96,7 +96,11 @@ func (b *RopeBuffer) Line(line int) []byte {
 // inclusive bounds. The returned value may or may not be a copy of the data,
 // so do not write to it.
 func (b *RopeBuffer) Slice(startLine, startCol, endLine, endCol int) []byte {
-	return (*rope.Node)(b).Slice(b.pos(startLine, startCol), b.pos(endLine, endCol)+1)
+	endPos := b.pos(endLine, endCol)+1
+	if length := (*rope.Node)(b).Len(); endPos >= length {
+		endPos = length-1
+	}
+	return (*rope.Node)(b).Slice(b.pos(startLine, startCol), endPos)
 }
 
 // Bytes returns all of the bytes in the buffer. This function is very likely

ui/textedit.go

@@ -94,13 +94,21 @@ loop:
 	lang := &buffer.Language {
 		Name: "Go",
 		Filetypes: []string{".go"},
-		Rules: map[*regexp.Regexp]buffer.Syntax {
+		Rules: map[*buffer.RegexpRegion]buffer.Syntax {
-			regexp.MustCompile("\\/\\/.*"): buffer.Comment,
+			&buffer.RegexpRegion{Start: regexp.MustCompile("\\/\\/.*")}: buffer.Comment,
-			regexp.MustCompile("\".*\""): buffer.String,
+			&buffer.RegexpRegion{Start: regexp.MustCompile("\".*\"")}: buffer.String,
-			regexp.MustCompile("\\b(var|if|else|range|for|switch|case|go|func|return|defer|import|package)\\b"): buffer.Keyword,
+			&buffer.RegexpRegion{
-			regexp.MustCompile("\\b(int|byte|string|bool)\\b"): buffer.Type,
+				Start: regexp.MustCompile("\\b(var|const|if|else|range|for|switch|case|go|func|return|defer|import|type|package)\\b"),
-			regexp.MustCompile("\\b([1-9][0-9]*|0[0-7]*|0[Xx][0-9A-Fa-f]+|0[Bb][01]+)\\b"): buffer.Number,
+			}: buffer.Keyword,
-			regexp.MustCompile("\\b(len|cap|panic)\\b"): buffer.Builtin,
+			&buffer.RegexpRegion{
+				Start: regexp.MustCompile("\\b(int|byte|string|bool|struct)\\b"),
+			}: buffer.Type,
+			&buffer.RegexpRegion{
+				Start: regexp.MustCompile("\\b([1-9][0-9]*|0[0-7]*|0[Xx][0-9A-Fa-f]+|0[Bb][01]+)\\b"),
+			}: buffer.Number,
+			&buffer.RegexpRegion{
+				Start: regexp.MustCompile("\\b(len|cap|panic)\\b"),
+			}: buffer.Builtin,
 		},
 	}
 
@@ -115,7 +123,6 @@ loop:
 	}
 
 	t.Highlighter = buffer.NewHighlighter(t.Buffer, lang, colorscheme)
-	t.Highlighter.Update()
 }
 
 // GetLineDelimiter returns "\r\n" for a CRLF buffer, or "\n" for an LF buffer.
@@ -144,6 +151,9 @@ func (t *TextEdit) ChangeLineDelimiters(crlf bool) {
 func (t *TextEdit) Delete(forwards bool) {
 	t.Dirty = true
 
+	var deletedLine bool // Whether any whole line has been deleted (changing the # of lines)
+	startingLine := t.cury
+
 	if t.selectMode { // If text is selected, delete the whole selection
 		t.selectMode = false // Disable selection and prevent infinite loop
 
@@ -151,20 +161,33 @@ func (t *TextEdit) Delete(forwards bool) {
 		t.Buffer.Remove(t.selection.StartLine, t.selection.StartCol, t.selection.EndLine, t.selection.EndCol)
 		t.SetLineCol(t.selection.StartLine, t.selection.StartCol) // Set cursor to start of region
 
-		return
+		deletedLine = t.selection.StartLine != t.selection.EndLine
+	} else { // Not deleting selection
+		if forwards { // Delete the character after the cursor
+			// If the cursor is not at the end of the last line...
+			if t.cury < t.Buffer.Lines()-1 || t.curx < t.Buffer.RunesInLine(t.cury) {
+				bytes := t.Buffer.Slice(t.cury, t.curx, t.cury, t.curx) // Get the character at cursor
+				deletedLine = bytes[0] == '\n'
+
+				t.Buffer.Remove(t.cury, t.curx, t.cury, t.curx) // Remove character at cursor
+			}
+		} else { // Delete the character before the cursor
+			// If the cursor is not at the first column of the first line...
+			if t.cury > 0 || t.curx > 0 {
+				t.CursorLeft() // Back up to that character
+
+				bytes := t.Buffer.Slice(t.cury, t.curx, t.cury, t.curx) // Get the char at cursor
+				deletedLine = bytes[0] == '\n'
+
+				t.Buffer.Remove(t.cury, t.curx, t.cury, t.curx) // Remove character at cursor
+			}
+		}
 	}
 
-	if forwards { // Delete the character after the cursor
+	if deletedLine {
-		// If the cursor is not at the end of the last line...
+		t.Highlighter.InvalidateLines(startingLine, t.Buffer.Lines()-1)
-		if t.cury < t.Buffer.Lines()-1 || t.curx < t.Buffer.RunesInLine(t.cury) {
+	} else {
-			t.Buffer.Remove(t.cury, t.curx, t.cury, t.curx) // Remove character at cursor
+		t.Highlighter.InvalidateLines(startingLine, startingLine)
-		}
-	} else { // Delete the character before the cursor
-		// If the cursor is not at the first column of the first line...
-		if t.cury > 0 || t.curx > 0 {
-			t.CursorLeft() // Back up to that character
-			t.Buffer.Remove(t.cury, t.curx, t.cury, t.curx) // Remove character at cursor
-		}
 	}
 }
 
@@ -178,6 +201,9 @@ func (t *TextEdit) Insert(contents string) {
 		t.Delete(true) // The parameter doesn't matter with selection
 	}
 
+	var lineInserted bool // True if contents contains a '\n'
+	startingLine := t.cury
+
 	runes := []rune(contents)
 	for i := 0; i < len(runes); i++ {
 		ch := runes[i]
@@ -188,10 +214,12 @@ func (t *TextEdit) Insert(contents string) {
 				i++ // Consume '\n' after
 				t.Buffer.Insert(t.cury, t.curx, []byte{'\n'})
 				t.SetLineCol(t.cury+1, 0) // Go to the start of that new line
+				lineInserted = true
 			}
 		case '\n':
 			t.Buffer.Insert(t.cury, t.curx, []byte{'\n'})
 			t.SetLineCol(t.cury+1, 0) // Go to the start of that new line
+			lineInserted = true
 		case '\b':
 			t.Delete(false) // Delete the character before the cursor
 		case '\t':
@@ -210,6 +238,12 @@ func (t *TextEdit) Insert(contents string) {
 		}
 	}
 	t.prevCurCol = t.curx
+
+	if lineInserted {
+		t.Highlighter.InvalidateLines(startingLine, t.Buffer.Lines()-1)
+	} else {
+		t.Highlighter.InvalidateLines(startingLine, startingLine)
+	}
 }
 
 // getTabCountInLineAtCol returns tabs in the given line, before the column position,
@@ -348,7 +382,7 @@ func (t *TextEdit) Draw(s tcell.Screen) {
 	selectedStyle := t.Theme.GetOrDefault("TextEditSelected")
 	columnStyle := t.Theme.GetOrDefault("TextEditColumn")
 
-	//DrawRect(s, t.x, t.y, t.width, t.height, ' ', textEditStyle) // Fill background
+	t.Highlighter.UpdateInvalidatedLines(t.scrolly, t.scrolly + (t.height-1))
 
 	var tabBytes []byte
 	if t.UseHardTabs {
@@ -368,11 +402,22 @@ func (t *TextEdit) Draw(s tcell.Screen) {
 			lineNumStr = strconv.Itoa(line + 1) // Only set for lines within the buffer (not view)
 
 			var lineBytes []byte = t.Buffer.Line(line) // Line to be drawn
+			var lineTabs  [128]int // Rune index for each hard tab '\t' in lineBytes
+			var tabs      int // Length of lineTabs (number of hard tabs)
 			if t.UseHardTabs {
+				var i int
+				for i < len(lineBytes) {
+					r, size := utf8.DecodeRune(lineBytes[i:])
+					if r == '\t' {
+						lineTabs[tabs] = i
+						tabs++
+					}
+					i += size
+				}
 				lineBytes = bytes.ReplaceAll(lineBytes, []byte{'\t'}, tabBytes)
 			}
 
-			lineHighlightData := t.Highlighter.GetLine(line)
+			lineHighlightData := t.Highlighter.GetLineMatches(line)
 			var lineHighlightDataIdx int
 
 			var byteIdx int      // Byte index of lineStr
@@ -387,12 +432,23 @@ func (t *TextEdit) Draw(s tcell.Screen) {
 				runeIdx++
 			}
 
+			tabOffsetAtRuneIdx := func(idx int) int {
+				var count int
+				for i := range lineTabs {
+					if i >= tabs || lineTabs[i] >= idx {
+						break
+					}
+					count++
+				}
+				return count * (t.TabSize - 1)
+			}
+
 			for col < t.x + t.width { // For each column in view...
 				var r rune = ' ' // Rune to draw this iteration
 				var size int = 1 // Size of the rune (in bytes)
 				var selected bool // Whether this rune should be styled as selected
 
-				tabOffsetAtRuneIdx := t.getTabCountInLineAtCol(line, runeIdx) * (t.TabSize-1)
+				tabOffsetAtRuneIdx := tabOffsetAtRuneIdx(runeIdx)
 
 				if byteIdx < len(lineBytes) { // If we are drawing part of the line contents...
 					r, size = utf8.DecodeRune(lineBytes[byteIdx:])