perf/buffers (#436)

* feat: add profiling for buffer mounting; close #231 * fix: add profiling script to makefile * perf: lazy load query editor * fix: improve waiting for editor in tests
tconbeer · Jan 30, 2024 · a78944f · a78944f
1 parent 6866c0f
commit a78944f
Show file tree

Hide file tree

Showing 21 changed files with 515 additions and 311 deletions.
diff --git a/.gitignore b/.gitignore
@@ -13,6 +13,7 @@
 Pipfile
 snapshot_report.html
 .harlequin.toml
+.profiles
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -4,21 +4,21 @@ repos:
     hooks:
       - id: black
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.6
+    rev: v0.1.15
     hooks:
       - id: ruff
         args: [ --fix, --exit-non-zero-on-fix ]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.7.1
+    rev: v1.8.0
     hooks:
       - id: mypy
         additional_dependencies:
           - click
           - duckdb>=0.8.0
           - shandy-sqlfmt[jinjafmt]
-          - textual>=0.41.0
-          - textual-textarea>=0.9.0
-          - textual-fastdatatable>=0.5.0
+          - textual>=0.47.1
+          - textual-textarea>=0.10.0
+          - textual-fastdatatable>=0.6.0
           - pytest
           - types-pygments
           - rich-click>=1.7.1

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+### Performance
+
+-   Harlequin now starts much faster, especially when restoring multiple buffers from the cache.
+
 ## [1.13.0] - 2024-01-26
 
 ### Features

diff --git a/Makefile b/Makefile
@@ -26,3 +26,8 @@ static/themes/%.svg: pyproject.toml src/scripts/export_screenshots.py
 
 static/harlequin.gif: static/harlequin.mp4
 	ffmpeg -i static/harlequin.mp4 -vf "fps=24,scale=640:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse" -loop 0 static/harlequin.gif
+
+profiles: .profiles/buffers.html
+
+.profiles/buffers.html: src/scripts/profile_buffers.py pyproject.toml $(shell find src/harlequin -type f)
+	pyinstrument -r html -o .profiles/buffers.html "src/scripts/profile_buffers.py"
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -21,9 +21,9 @@ build-backend = "poetry.core.masonry.api"
 python = ">=3.8.1,<4.0.0"
 
 # textual and component libraries
-textual = "==0.46.0"
+textual = "==0.47.1"
 textual-fastdatatable = "==0.6.3"
-textual-textarea = "==0.9.5"
+textual-textarea = "==0.10.0"
 
 # click
 click = "^8.1.3"
@@ -33,7 +33,6 @@ rich-click = "^1.7.1"
 duckdb = ">=0.8.0"
 shandy-sqlfmt = ">=0.19.0"
 platformdirs = ">=3.10,<5.0"
-pyperclip = "^1.8.2"
 importlib_metadata = { version = ">=4.6.0", python = "<3.10.0" }
 tomli = { version = "^2.0.1", python = "<3.11.0" }
 tomlkit = "^0.12.3"
@@ -55,6 +54,7 @@ textual-dev = "^1.0.1"
 harlequin-postgres = "^0.2"
 harlequin-mysql = "^0.1.1"
 harlequin-odbc = "^0.1.1"
+pyinstrument = "^4.6.2"
 
 [tool.poetry.group.static.dependencies]
 black = "^23.3.0"

diff --git a/src/harlequin/app.py b/src/harlequin/app.py
@@ -14,6 +14,7 @@
 from textual.css.stylesheet import Stylesheet
 from textual.dom import DOMNode
 from textual.driver import Driver
+from textual.lazy import Lazy
 from textual.message import Message
 from textual.reactive import reactive
 from textual.screen import Screen, ScreenResultCallbackType, ScreenResultType
@@ -191,7 +192,13 @@ def compose(self) -> ComposeResult:
             show_files=self.show_files,
             show_s3=self.show_s3,
         )
-        self.editor_collection = EditorCollection(language="sql", theme=self.theme)
+        self.editor_collection = EditorCollection(
+            language="sql", theme=self.theme, classes="hide-tabs"
+        )
+        self.editor: CodeEditor | None = None
+        editor_placeholder = Lazy(widget=self.editor_collection)
+        editor_placeholder.border_title = self.editor_collection.border_title
+        editor_placeholder.loading = True
         self.results_viewer = ResultsViewer(
             max_results=self.max_results,
             type_color=self.app_colors.gray,
@@ -205,7 +212,7 @@ def compose(self) -> ComposeResult:
         with Horizontal():
             yield self.data_catalog
             with Vertical(id="main_panel"):
-                yield self.editor_collection
+                yield editor_placeholder
                 yield self.run_query_bar
                 yield self.results_viewer
         yield self.footer
@@ -216,7 +223,7 @@ def push_screen(  # type: ignore
         callback: Union[ScreenResultCallbackType[ScreenResultType], None] = None,
         wait_for_dismiss: bool = False,
     ) -> Union[AwaitMount, asyncio.Future[ScreenResultType]]:
-        if self.editor._has_focus_within:
+        if self.editor is not None and self.editor._has_focus_within:
             self.editor.text_input._pause_blink(visible=True)
         return super().push_screen(  # type: ignore
             screen,
@@ -226,7 +233,11 @@ def push_screen(  # type: ignore
 
     def pop_screen(self) -> Screen[object]:
         new_screen = super().pop_screen()
-        if len(self.screen_stack) == 1 and self.editor._has_focus_within:
+        if (
+            len(self.screen_stack) == 1
+            and self.editor is not None
+            and self.editor._has_focus_within
+        ):
             self.editor.text_input._restart_blink()
         return new_screen
 
@@ -240,8 +251,6 @@ def append_to_history(
         )
 
     async def on_mount(self) -> None:
-        self.editor = self.editor_collection.current_editor
-        self.editor.focus()
         self.run_query_bar.checkbox.value = False
 
         self._connect()
@@ -292,14 +301,25 @@ def initialize_app(self, message: DatabaseConnected) -> None:
     @on(DataCatalog.NodeSubmitted)
     def insert_node_into_editor(self, message: DataCatalog.NodeSubmitted) -> None:
         message.stop()
+        if self.editor is None:
+            # recycle message while editor loads
+            self.post_message(message=message)
+            return
         self.editor.insert_text_at_selection(text=message.insert_name)
         self.editor.focus()
 
     @on(DataCatalog.NodeCopied)
     def copy_node_name(self, message: DataCatalog.NodeCopied) -> None:
         message.stop()
+        if self.editor is None:
+            # recycle message while we wait for the editor to load
+            self.post_message(message=message)
+            return
         self.editor.text_input.clipboard = message.copy_name
-        if self.editor.use_system_clipboard:
+        if (
+            self.editor.use_system_clipboard
+            and self.editor.text_input.system_copy is not None
+        ):
             try:
                 self.editor.text_input.system_copy(message.copy_name)
             except Exception:
@@ -355,10 +375,17 @@ def submit_query_if_limit_valid(self, message: Input.Submitted) -> None:
     @on(DataTable.SelectionCopied)
     def copy_data_to_clipboard(self, message: DataTable.SelectionCopied) -> None:
         message.stop()
+        if self.editor is None:
+            # recycle the message while we wait for the editor to load
+            self.post_message(message=message)
+            return
         # Excel, sheets, and Snowsight all use a TSV format for copying tabular data
         text = os.linesep.join("\t".join(map(str, row)) for row in message.values)
         self.editor.text_input.clipboard = text
-        if self.editor.use_system_clipboard:
+        if (
+            self.editor.use_system_clipboard
+            and self.editor.text_input.system_copy is not None
+        ):
             try:
                 self.editor.text_input.system_copy(text)
             except Exception:
@@ -524,7 +551,7 @@ def execute_query(self, message: QuerySubmitted) -> None:
 
     def watch_sidebar_hidden(self, sidebar_hidden: bool) -> None:
         if sidebar_hidden:
-            if self.data_catalog.has_focus:
+            if self.data_catalog.has_focus and self.editor is not None:
                 self.editor.focus()
         self.data_catalog.disabled = sidebar_hidden
 
@@ -599,7 +626,8 @@ def action_focus_data_catalog(self) -> None:
         self.data_catalog.focus()
 
     def action_focus_query_editor(self) -> None:
-        self.editor.focus()
+        if self.editor is not None:
+            self.editor.focus()
 
     def action_focus_results_viewer(self) -> None:
         self.results_viewer.focus()
@@ -705,6 +733,8 @@ def _execute_query(self, message: QuerySubmitted) -> None:
         )
 
     def _get_query_text(self) -> str:
+        if self.editor is None:
+            return ""
         return (
             self._validate_selection()
             or self.editor.current_query
@@ -788,6 +818,8 @@ def _validate_selection(self) -> str:
         If the selection is valid query, return it. Otherwise
         return the empty string.
         """
+        if self.editor is None:
+            return ""
         selection = self.editor.selected_text
         if self.connection is None:
             return selection

diff --git a/src/harlequin/colors.py b/src/harlequin/colors.py
@@ -1,4 +1,4 @@
-from typing import Dict, Union
+from typing import Dict, Type, Union
 
 from pygments.style import Style as PygmentsStyle
 from pygments.styles import get_style_by_name
@@ -163,18 +163,23 @@ def __init__(
 
     @classmethod
     def from_theme(cls, theme: str) -> "HarlequinColors":
-        try:
-            style = get_style_by_name(theme)
-        except ClassNotFound as e:
-            raise HarlequinThemeError(
-                (
-                    f"No theme found with the name {theme}.\n"
-                    "Theme must be the name of a Pygments Style. "
-                    "You can browse the supported styles here:\n"
-                    "https://pygments.org/styles/"
-                ),
-                title="Harlequin couldn't load your theme.",
-            ) from e
+        # perf optimization to short-circuit get_style_by_name call, which
+        # is slow.
+        if theme == "harlequin":
+            style: Type[PygmentsStyle] = HarlequinPygmentsStyle
+        else:
+            try:
+                style = get_style_by_name(theme)
+            except ClassNotFound as e:
+                raise HarlequinThemeError(
+                    (
+                        f"No theme found with the name {theme}.\n"
+                        "Theme must be the name of a Pygments Style. "
+                        "You can browse the supported styles here:\n"
+                        "https://pygments.org/styles/"
+                    ),
+                    title="Harlequin couldn't load your theme.",
+                ) from e
 
         background = style.background_color
         highlight = style.highlight_color

diff --git a/src/harlequin/components/code_editor.py b/src/harlequin/components/code_editor.py
@@ -203,7 +203,10 @@ def member_completer(self) -> MemberCompleter | None:
     @member_completer.setter
     def member_completer(self, new_completer: MemberCompleter) -> None:
         self._member_completer = new_completer
-        self.current_editor.member_completer = new_completer
+        try:
+            self.current_editor.member_completer = new_completer
+        except NoMatches:
+            pass
 
     @property
     def word_completer(self) -> WordCompleter | None:
@@ -212,10 +215,12 @@ def word_completer(self) -> WordCompleter | None:
     @word_completer.setter
     def word_completer(self, new_completer: WordCompleter) -> None:
         self._word_completer = new_completer
-        self.current_editor.word_completer = new_completer
+        try:
+            self.current_editor.word_completer = new_completer
+        except NoMatches:
+            pass
 
     async def on_mount(self) -> None:
-        self.add_class("hide-tabs")
         cache = load_cache()
         if cache is not None:
             for _i, buffer in enumerate(cache.buffers):
@@ -226,6 +231,8 @@ async def on_mount(self) -> None:
         else:
             await self.action_new_buffer()
         self.query_one(Tabs).can_focus = False
+        self.current_editor.word_completer = self.word_completer
+        self.current_editor.member_completer = self.member_completer
 
     def on_focus(self) -> None:
         self.current_editor.focus()
@@ -251,6 +258,7 @@ async def action_new_buffer(
         new_tab_id = f"tab-{self.counter}"
         editor = CodeEditor(
             id=f"buffer-{self.counter}",
+            text=state.text if state is not None else "",
             language=self.language,
             theme=self.theme,
             word_completer=self.word_completer,
@@ -263,12 +271,14 @@ async def action_new_buffer(
         )
         await self.add_pane(pane)  # type: ignore
         if state is not None:
-            editor.text = state.text
             editor.cursor = state.cursor
             editor.selection_anchor = state.selection_anchor
         else:
             self.active = new_tab_id
-            self.current_editor.focus()
+            try:
+                self.current_editor.focus()
+            except NoMatches:
+                pass
         if self.counter > 1:
             self.remove_class("hide-tabs")
         return editor

diff --git a/src/harlequin/global.tcss b/src/harlequin/global.tcss
@@ -7,7 +7,8 @@ $border-title-color-focus: $primary;
 /* ALL WIDGETS */
 DataCatalog,
 EditorCollection,
-ResultsViewer {
+ResultsViewer,
+Lazy {
     border: round $border-color-nofocus;
     border-title-color: $border-title-color-nofocus;
     background: $background;
@@ -53,6 +54,14 @@ Vertical:disabled {
     border: none;
 }
 
+Lazy {
+    background: $background;
+    color: $background;
+    display: block;
+    height: 1fr;
+    width: 100%;
+}
+
 Toast {
     width: auto;
     background: $background;

diff --git a/src/scripts/export_screenshots.py b/src/scripts/export_screenshots.py
@@ -25,11 +25,14 @@ async def save_all_screenshots() -> None:
         print(f"Screenshotting {theme}")
         app = Harlequin(adapter=adapter, theme=theme)
         async with app.run_test(size=(120, 36)) as pilot:
+            await app.workers.wait_for_complete()
+            await pilot.pause()
+            if app.editor is None:
+                await pilot.pause(0.2)
+            assert app.editor is not None
             app.editor.text = TEXT
             app.editor.cursor = (9, 16)  # type: ignore
             app.editor.selection_anchor = (9, 0)  # type: ignore
-            await app.workers.wait_for_complete()
-            await pilot.pause()
             app.data_catalog.database_tree.root.expand()
             for child in app.data_catalog.database_tree.root.children:
                 print("here!")