From dda68fbb5da4b70cf67fd0f825e676d015cfca8c Mon Sep 17 00:00:00 2001 From: Loren Siebert Date: Fri, 31 Mar 2023 17:04:10 -0700 Subject: [PATCH 1/2] Propagate exceptions from `ThreadPoolExecutor` - Non-recoverable errors (e.g., missing materialized view) in thread tasks setup() and refresh_view() should bubble up to the application and cause it to exit. --- pyproject.toml | 1 + src/sinker/runner.py | 14 +++++++++++--- src/sinker/sinker.py | 6 ++++-- tests/test_e2e.py | 20 ++++++++++++++++++++ 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9c73b91..495a369 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ python = "^3.9" elasticsearch = "^8.6.1" environs = "^9.5.0" psycopg = "^3.1.8" +pytest-mock = "^3.10.0" [tool.poetry.group.dev.dependencies] flake8 = "^6.0.0" diff --git a/src/sinker/runner.py b/src/sinker/runner.py index 907ebd3..054fb78 100644 --- a/src/sinker/runner.py +++ b/src/sinker/runner.py @@ -1,3 +1,4 @@ +import concurrent import json import logging from concurrent.futures import ThreadPoolExecutor @@ -41,8 +42,12 @@ class Runner: # set up materialized views and Elasticsearch indices, and populate them with initial data with ThreadPoolExecutor(max_workers=len(self.views_to_sinkers)) as executor: + futures = [] for sinker in self.views_to_sinkers.values(): - executor.submit(sinker.setup) + futures.append(executor.submit(sinker.setup)) + for future in concurrent.futures.as_completed(futures): + view: str = future.result() + logger.info(f"{view} sinker is set up") parent_tables_to_indices: dict[str, str] = { sinker.parent_table: sinker.index for sinker in self.views_to_sinkers.values() @@ -84,11 +89,14 @@ class Runner: sleep(SINKER_POLL_INTERVAL) return with ThreadPoolExecutor(max_workers=len(views)) as executor: + futures = [] for view_tuple in views: view: str = view_tuple[0].split(SCHEMA_TABLE_DELIMITER)[1] sinker: Sinker = self.views_to_sinkers[view] - executor.submit(sinker.refresh_view) - logger.debug(f"{view} view is refreshed") + futures.append(executor.submit(sinker.refresh_view)) + for future in concurrent.futures.as_completed(futures): + view: str = future.result() + logger.info(f"{view} view is refreshed") # --------------------------------------- # a triggered update from here on will cause a new materialized view entry to be added to the "end" diff --git a/src/sinker/sinker.py b/src/sinker/sinker.py index 0acb9ca..70b1ba6 100644 --- a/src/sinker/sinker.py +++ b/src/sinker/sinker.py @@ -35,12 +35,13 @@ class Sinker: self.index: str = index self.parent_table: str = "" # defined during setup process - def setup(self): + def setup(self) -> str: """ Creates materialized views with supporting functions and triggers, and creates Elasticsearch indices """ self.setup_pg() self.setup_es() + return self.view def setup_es(self) -> None: logger.info(f"Setting up the {self.index} Elasticsearch index") @@ -127,7 +128,8 @@ class Sinker: # 0/24EF4718,17394,COMMIT 17394 self.parent_table = schema_tables[-1] - def refresh_view(self) -> None: + def refresh_view(self) -> str: logger.info(f"Refreshing the {self.view} materialized view") refresh_view_query: str = q.REFRESH_VIEW.format(SINKER_SCHEMA, self.view) psycopg.connect(autocommit=True).execute(refresh_view_query) + return self.view diff --git a/tests/test_e2e.py b/tests/test_e2e.py index d3cac26..e753606 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -32,6 +32,26 @@ def wait_for_pg_es(): sleep(1) +def test_exception_propagation_during_refresh(wait_for_pg_es): + runner = Runner() + # Update a record in Postgres but remove the materialized view so it errors when we iterate + conn = psycopg.connect(autocommit=True) + cur = conn.cursor() + cur.execute("update public.person set name = 'Uh oh' where id = 'p-1'") + cur.execute("drop materialized view sinker.person_mv") + # make sure the exception gets raised from the thread versus silently consuming it + with pytest.raises(psycopg.errors.UndefinedTable): + runner.iterate() + + +def test_exception_propagation_during_setup(wait_for_pg_es, mocker): + # mock es.get_client() to raise an exception in the thread that sets up the sinker + mocker.patch("sinker.sinker.get_client", side_effect=elasticsearch.exceptions.ConnectionError('Boom!')) + # make sure the exception gets raised from the thread versus silently consuming it + with pytest.raises(elasticsearch.exceptions.ConnectionError): + Runner() + + def test_end_to_end(wait_for_pg_es): runner = Runner() es = get_client() From 4e00a91eb16335ea67b8d3fd9f60ce7de55885a6 Mon Sep 17 00:00:00 2001 From: Loren Siebert Date: Sat, 1 Apr 2023 08:10:38 -0700 Subject: [PATCH 2/2] mypy fix --- src/sinker/runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sinker/runner.py b/src/sinker/runner.py index 054fb78..617d50b 100644 --- a/src/sinker/runner.py +++ b/src/sinker/runner.py @@ -95,8 +95,8 @@ class Runner: sinker: Sinker = self.views_to_sinkers[view] futures.append(executor.submit(sinker.refresh_view)) for future in concurrent.futures.as_completed(futures): - view: str = future.result() - logger.info(f"{view} view is refreshed") + view_result: str = future.result() + logger.info(f"{view_result} view is refreshed") # --------------------------------------- # a triggered update from here on will cause a new materialized view entry to be added to the "end"