From dda68fbb5da4b70cf67fd0f825e676d015cfca8c Mon Sep 17 00:00:00 2001 From: Loren Siebert Date: Fri, 31 Mar 2023 17:04:10 -0700 Subject: [PATCH 01/20] Propagate exceptions from `ThreadPoolExecutor` - Non-recoverable errors (e.g., missing materialized view) in thread tasks setup() and refresh_view() should bubble up to the application and cause it to exit. --- pyproject.toml | 1 + src/sinker/runner.py | 14 +++++++++++--- src/sinker/sinker.py | 6 ++++-- tests/test_e2e.py | 20 ++++++++++++++++++++ 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9c73b91..495a369 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ python = "^3.9" elasticsearch = "^8.6.1" environs = "^9.5.0" psycopg = "^3.1.8" +pytest-mock = "^3.10.0" [tool.poetry.group.dev.dependencies] flake8 = "^6.0.0" diff --git a/src/sinker/runner.py b/src/sinker/runner.py index 907ebd3..054fb78 100644 --- a/src/sinker/runner.py +++ b/src/sinker/runner.py @@ -1,3 +1,4 @@ +import concurrent import json import logging from concurrent.futures import ThreadPoolExecutor @@ -41,8 +42,12 @@ class Runner: # set up materialized views and Elasticsearch indices, and populate them with initial data with ThreadPoolExecutor(max_workers=len(self.views_to_sinkers)) as executor: + futures = [] for sinker in self.views_to_sinkers.values(): - executor.submit(sinker.setup) + futures.append(executor.submit(sinker.setup)) + for future in concurrent.futures.as_completed(futures): + view: str = future.result() + logger.info(f"{view} sinker is set up") parent_tables_to_indices: dict[str, str] = { sinker.parent_table: sinker.index for sinker in self.views_to_sinkers.values() @@ -84,11 +89,14 @@ class Runner: sleep(SINKER_POLL_INTERVAL) return with ThreadPoolExecutor(max_workers=len(views)) as executor: + futures = [] for view_tuple in views: view: str = view_tuple[0].split(SCHEMA_TABLE_DELIMITER)[1] sinker: Sinker = self.views_to_sinkers[view] - executor.submit(sinker.refresh_view) - logger.debug(f"{view} view is refreshed") + futures.append(executor.submit(sinker.refresh_view)) + for future in concurrent.futures.as_completed(futures): + view: str = future.result() + logger.info(f"{view} view is refreshed") # --------------------------------------- # a triggered update from here on will cause a new materialized view entry to be added to the "end" diff --git a/src/sinker/sinker.py b/src/sinker/sinker.py index 0acb9ca..70b1ba6 100644 --- a/src/sinker/sinker.py +++ b/src/sinker/sinker.py @@ -35,12 +35,13 @@ class Sinker: self.index: str = index self.parent_table: str = "" # defined during setup process - def setup(self): + def setup(self) -> str: """ Creates materialized views with supporting functions and triggers, and creates Elasticsearch indices """ self.setup_pg() self.setup_es() + return self.view def setup_es(self) -> None: logger.info(f"Setting up the {self.index} Elasticsearch index") @@ -127,7 +128,8 @@ class Sinker: # 0/24EF4718,17394,COMMIT 17394 self.parent_table = schema_tables[-1] - def refresh_view(self) -> None: + def refresh_view(self) -> str: logger.info(f"Refreshing the {self.view} materialized view") refresh_view_query: str = q.REFRESH_VIEW.format(SINKER_SCHEMA, self.view) psycopg.connect(autocommit=True).execute(refresh_view_query) + return self.view diff --git a/tests/test_e2e.py b/tests/test_e2e.py index d3cac26..e753606 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -32,6 +32,26 @@ def wait_for_pg_es(): sleep(1) +def test_exception_propagation_during_refresh(wait_for_pg_es): + runner = Runner() + # Update a record in Postgres but remove the materialized view so it errors when we iterate + conn = psycopg.connect(autocommit=True) + cur = conn.cursor() + cur.execute("update public.person set name = 'Uh oh' where id = 'p-1'") + cur.execute("drop materialized view sinker.person_mv") + # make sure the exception gets raised from the thread versus silently consuming it + with pytest.raises(psycopg.errors.UndefinedTable): + runner.iterate() + + +def test_exception_propagation_during_setup(wait_for_pg_es, mocker): + # mock es.get_client() to raise an exception in the thread that sets up the sinker + mocker.patch("sinker.sinker.get_client", side_effect=elasticsearch.exceptions.ConnectionError('Boom!')) + # make sure the exception gets raised from the thread versus silently consuming it + with pytest.raises(elasticsearch.exceptions.ConnectionError): + Runner() + + def test_end_to_end(wait_for_pg_es): runner = Runner() es = get_client() From 4e00a91eb16335ea67b8d3fd9f60ce7de55885a6 Mon Sep 17 00:00:00 2001 From: Loren Siebert Date: Sat, 1 Apr 2023 08:10:38 -0700 Subject: [PATCH 02/20] mypy fix --- src/sinker/runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sinker/runner.py b/src/sinker/runner.py index 054fb78..617d50b 100644 --- a/src/sinker/runner.py +++ b/src/sinker/runner.py @@ -95,8 +95,8 @@ class Runner: sinker: Sinker = self.views_to_sinkers[view] futures.append(executor.submit(sinker.refresh_view)) for future in concurrent.futures.as_completed(futures): - view: str = future.result() - logger.info(f"{view} view is refreshed") + view_result: str = future.result() + logger.info(f"{view_result} view is refreshed") # --------------------------------------- # a triggered update from here on will cause a new materialized view entry to be added to the "end" From ba05ff344a604a7f4b7a643911f9dd87ee7724c5 Mon Sep 17 00:00:00 2001 From: Loren Siebert Date: Sat, 1 Apr 2023 08:52:56 -0700 Subject: [PATCH 03/20] Propagate exceptions from `ThreadPoolExecutor` (#3) Non-recoverable errors (e.g., missing materialized view) in thread tasks setup() and refresh_view() should bubble up to the application and cause it to exit. --- pyproject.toml | 1 + src/sinker/runner.py | 14 +++++++++++--- src/sinker/sinker.py | 6 ++++-- tests/test_e2e.py | 20 ++++++++++++++++++++ 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9c73b91..495a369 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ python = "^3.9" elasticsearch = "^8.6.1" environs = "^9.5.0" psycopg = "^3.1.8" +pytest-mock = "^3.10.0" [tool.poetry.group.dev.dependencies] flake8 = "^6.0.0" diff --git a/src/sinker/runner.py b/src/sinker/runner.py index 907ebd3..617d50b 100644 --- a/src/sinker/runner.py +++ b/src/sinker/runner.py @@ -1,3 +1,4 @@ +import concurrent import json import logging from concurrent.futures import ThreadPoolExecutor @@ -41,8 +42,12 @@ class Runner: # set up materialized views and Elasticsearch indices, and populate them with initial data with ThreadPoolExecutor(max_workers=len(self.views_to_sinkers)) as executor: + futures = [] for sinker in self.views_to_sinkers.values(): - executor.submit(sinker.setup) + futures.append(executor.submit(sinker.setup)) + for future in concurrent.futures.as_completed(futures): + view: str = future.result() + logger.info(f"{view} sinker is set up") parent_tables_to_indices: dict[str, str] = { sinker.parent_table: sinker.index for sinker in self.views_to_sinkers.values() @@ -84,11 +89,14 @@ class Runner: sleep(SINKER_POLL_INTERVAL) return with ThreadPoolExecutor(max_workers=len(views)) as executor: + futures = [] for view_tuple in views: view: str = view_tuple[0].split(SCHEMA_TABLE_DELIMITER)[1] sinker: Sinker = self.views_to_sinkers[view] - executor.submit(sinker.refresh_view) - logger.debug(f"{view} view is refreshed") + futures.append(executor.submit(sinker.refresh_view)) + for future in concurrent.futures.as_completed(futures): + view_result: str = future.result() + logger.info(f"{view_result} view is refreshed") # --------------------------------------- # a triggered update from here on will cause a new materialized view entry to be added to the "end" diff --git a/src/sinker/sinker.py b/src/sinker/sinker.py index 0acb9ca..70b1ba6 100644 --- a/src/sinker/sinker.py +++ b/src/sinker/sinker.py @@ -35,12 +35,13 @@ class Sinker: self.index: str = index self.parent_table: str = "" # defined during setup process - def setup(self): + def setup(self) -> str: """ Creates materialized views with supporting functions and triggers, and creates Elasticsearch indices """ self.setup_pg() self.setup_es() + return self.view def setup_es(self) -> None: logger.info(f"Setting up the {self.index} Elasticsearch index") @@ -127,7 +128,8 @@ class Sinker: # 0/24EF4718,17394,COMMIT 17394 self.parent_table = schema_tables[-1] - def refresh_view(self) -> None: + def refresh_view(self) -> str: logger.info(f"Refreshing the {self.view} materialized view") refresh_view_query: str = q.REFRESH_VIEW.format(SINKER_SCHEMA, self.view) psycopg.connect(autocommit=True).execute(refresh_view_query) + return self.view diff --git a/tests/test_e2e.py b/tests/test_e2e.py index d3cac26..e753606 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -32,6 +32,26 @@ def wait_for_pg_es(): sleep(1) +def test_exception_propagation_during_refresh(wait_for_pg_es): + runner = Runner() + # Update a record in Postgres but remove the materialized view so it errors when we iterate + conn = psycopg.connect(autocommit=True) + cur = conn.cursor() + cur.execute("update public.person set name = 'Uh oh' where id = 'p-1'") + cur.execute("drop materialized view sinker.person_mv") + # make sure the exception gets raised from the thread versus silently consuming it + with pytest.raises(psycopg.errors.UndefinedTable): + runner.iterate() + + +def test_exception_propagation_during_setup(wait_for_pg_es, mocker): + # mock es.get_client() to raise an exception in the thread that sets up the sinker + mocker.patch("sinker.sinker.get_client", side_effect=elasticsearch.exceptions.ConnectionError('Boom!')) + # make sure the exception gets raised from the thread versus silently consuming it + with pytest.raises(elasticsearch.exceptions.ConnectionError): + Runner() + + def test_end_to_end(wait_for_pg_es): runner = Runner() es = get_client() From 752171fc8b033b0791c7228fbca5e99cd2c4d476 Mon Sep 17 00:00:00 2001 From: Loren Siebert Date: Thu, 1 Jun 2023 11:19:15 -0700 Subject: [PATCH 04/20] Fix handling of function calls in materialized view queries (#4) --- pyproject.toml | 2 +- src/sinker/sinker.py | 28 +++++++++++++--------------- src/sinker/utils.py | 16 ++++++++++++++++ tests/test_generate_schema_tables.py | 12 ++++++++++++ 4 files changed, 42 insertions(+), 16 deletions(-) create mode 100644 src/sinker/utils.py create mode 100644 tests/test_generate_schema_tables.py diff --git a/pyproject.toml b/pyproject.toml index 495a369..e716255 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "sinker" -version = "0.1.0" +version = "0.1.1" description = "Synchronize Postgres to Elasticsearch" authors = ["Loren Siebert "] license = "MIT/Apache-2.0" diff --git a/src/sinker/sinker.py b/src/sinker/sinker.py index 70b1ba6..d8e4a25 100644 --- a/src/sinker/sinker.py +++ b/src/sinker/sinker.py @@ -1,7 +1,6 @@ import json import logging import os -import re from typing import Iterable, Dict, Any import psycopg @@ -18,11 +17,11 @@ from .settings import ( PGCHUNK_SIZE, SCHEMA_TABLE_DELIMITER, ) +from .utils import generate_schema_tables logger = logging.getLogger(__name__) BACKFILL_CURSOR_NAME = "backfill" -TABLE_RE = re.compile(r"from\s\"?(\S+)\b", re.I) class Sinker: @@ -106,27 +105,26 @@ class Sinker: plpgsql: str = f"{schema_view_name}_fn" create_function: str = q.CREATE_FUNCTION.format(plpgsql, SINKER_SCHEMA, SINKER_TODO_TABLE, schema_view_name) ddl_list.append(create_function) - schema_tables: list[Any] = TABLE_RE.findall(view_select_query) - for schema_table in schema_tables: + for schema_table in generate_schema_tables(view_select_query): schema, _, table = schema_table.rpartition(SCHEMA_TABLE_DELIMITER) schema = schema or DEFAULT_SCHEMA trigger_name: str = f"{SINKER_SCHEMA}_{self.view}_{schema}_{table}" create_trigger: str = q.CREATE_TRIGGER.format(trigger_name, schema, table, plpgsql) ddl_list.append(create_trigger) + # The last table is the top-level table that gets DELETE events with an ID in the replication slot. + # The materialized views do not contain the ID of the doc being deleted, + # so we'll use this table's delete events as a proxy. + # lsn,xid,data + # 0/24EDA4D8,17393,BEGIN 17393 + # 0/24EDA4D8,17393,"table public.""Foo"": DELETE: id[text]:'91754ea9-2983-4cf7-bdf9-fc23d2386d90'" + # 0/24EDC1B0,17393,COMMIT 17393 + # 0/24EDC228,17394,BEGIN 17394 + # 0/24EF0D60,17394,table sinker.foo_mv: DELETE: (no-tuple-data) + # 0/24EF4718,17394,COMMIT 17394 + self.parent_table = schema_table create_todo_entry: str = q.CREATE_TODO_ENTRY.format(SINKER_SCHEMA, SINKER_TODO_TABLE, schema_view_name) ddl_list.append(create_todo_entry) psycopg.connect(autocommit=True).execute("; ".join(ddl_list)) - # The last table is the top-level table that gets DELETE events with an ID in the replication slot. - # The materialized views do not contain the ID of the doc being deleted, - # so we'll use this table's delete events as a proxy. - # lsn,xid,data - # 0/24EDA4D8,17393,BEGIN 17393 - # 0/24EDA4D8,17393,"table public.""Foo"": DELETE: id[text]:'91754ea9-2983-4cf7-bdf9-fc23d2386d90'" - # 0/24EDC1B0,17393,COMMIT 17393 - # 0/24EDC228,17394,BEGIN 17394 - # 0/24EF0D60,17394,table sinker.foo_mv: DELETE: (no-tuple-data) - # 0/24EF4718,17394,COMMIT 17394 - self.parent_table = schema_tables[-1] def refresh_view(self) -> str: logger.info(f"Refreshing the {self.view} materialized view") diff --git a/src/sinker/utils.py b/src/sinker/utils.py new file mode 100644 index 0000000..4aa745a --- /dev/null +++ b/src/sinker/utils.py @@ -0,0 +1,16 @@ +import re + +from typing import Iterable + +TABLE_RE = re.compile(r"from\s\"?(\S+)\b", re.I) + + +def generate_schema_tables(view_select_query: str) -> Iterable[str]: + """ + Given a view select query, return a list of tables that are referenced in the query. + Skip anything that looks like a function call. + :param view_select_query: The select query from the view + """ + for table_candidate in TABLE_RE.findall(view_select_query): + if "(" not in table_candidate: + yield table_candidate diff --git a/tests/test_generate_schema_tables.py b/tests/test_generate_schema_tables.py new file mode 100644 index 0000000..1bf7d74 --- /dev/null +++ b/tests/test_generate_schema_tables.py @@ -0,0 +1,12 @@ +from sinker.utils import generate_schema_tables + + +def test_generate_schema_tables(): + view_select_query = """select id, + json_build_object( + 'name', "name", + 'emailDomains',(select array_agg(split_part(email, '@', 2)) FROM unnest(emails) as email), + ) as "person" + from "person" + """ + assert list(generate_schema_tables(view_select_query)) == ["person"] From 53ba0d0a5411b03fe129cf226e928b67ce51cf0f Mon Sep 17 00:00:00 2001 From: Loren Siebert Date: Mon, 11 Dec 2023 15:43:29 -0800 Subject: [PATCH 05/20] Dedupe entries when generating schema tables (#5) Something like this should only generate one table entry for `EmailAddress`: ``` 'emailDomains', (select array_agg(split_part(value, '@', 2)) from "EmailAddress" EA where "personId"="Person".id), 'emailAddresses', (select array_agg(value) from "EmailAddress" EA where "personId"="Person".id), ``` --- pyproject.toml | 2 +- src/sinker/utils.py | 8 ++++++-- tests/test_generate_schema_tables.py | 7 +++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e716255..16f8508 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "sinker" -version = "0.1.1" +version = "0.1.2" description = "Synchronize Postgres to Elasticsearch" authors = ["Loren Siebert "] license = "MIT/Apache-2.0" diff --git a/src/sinker/utils.py b/src/sinker/utils.py index 4aa745a..cfed5c9 100644 --- a/src/sinker/utils.py +++ b/src/sinker/utils.py @@ -7,10 +7,14 @@ TABLE_RE = re.compile(r"from\s\"?(\S+)\b", re.I) def generate_schema_tables(view_select_query: str) -> Iterable[str]: """ - Given a view select query, return a list of tables that are referenced in the query. + Given a view select query, return a list of unique tables that are referenced in the query + in the order they were encountered. Skip anything that looks like a function call. :param view_select_query: The select query from the view """ + seen: set = set() for table_candidate in TABLE_RE.findall(view_select_query): if "(" not in table_candidate: - yield table_candidate + if table_candidate not in seen: + seen.add(table_candidate) + yield table_candidate diff --git a/tests/test_generate_schema_tables.py b/tests/test_generate_schema_tables.py index 1bf7d74..ef8cb14 100644 --- a/tests/test_generate_schema_tables.py +++ b/tests/test_generate_schema_tables.py @@ -5,8 +5,11 @@ def test_generate_schema_tables(): view_select_query = """select id, json_build_object( 'name', "name", - 'emailDomains',(select array_agg(split_part(email, '@', 2)) FROM unnest(emails) as email), + 'otherEmailDomains',(select array_agg(split_part(email, '@', 2)) FROM unnest(emails) as email), + 'emailDomains', (select array_agg(split_part(value, '@', 2)) + from "EmailAddress" EA where "personId"="Person".id), + 'emailAddresses', (select array_agg(value) from "EmailAddress" EA where "personId"="Person".id), ) as "person" from "person" """ - assert list(generate_schema_tables(view_select_query)) == ["person"] + assert list(generate_schema_tables(view_select_query)) == ["EmailAddress", "person"] From 87a57ac3ed0284fd4409107d065abccc7d1751a0 Mon Sep 17 00:00:00 2001 From: Loren Siebert Date: Mon, 11 Dec 2023 15:47:51 -0800 Subject: [PATCH 06/20] Support Python 3.12 (#6) --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 32a41de..0f5abed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9", "3.10", "3.11" ] + python-version: [ "3.9", "3.10", "3.11", "3.12" ] steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -40,4 +40,4 @@ jobs: - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 with: - token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file + token: ${{ secrets.CODECOV_TOKEN }} From 8e31bb1963582dd48713e4c6e83ab55b209051b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Dec 2023 16:52:36 -0800 Subject: [PATCH 07/20] Update environs requirement from ^9.5.0 to >=9.5,<11.0 (#7) Updates the requirements on [environs](https://github.com/sloria/environs) to permit the latest version. - [Changelog](https://github.com/sloria/environs/blob/master/CHANGELOG.md) - [Commits](https://github.com/sloria/environs/compare/9.5.0...10.0.0) --- updated-dependencies: - dependency-name: environs dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 16f8508..613c803 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ sinker = "sinker.__main__:main" [tool.poetry.dependencies] python = "^3.9" elasticsearch = "^8.6.1" -environs = "^9.5.0" +environs = ">=9.5,<11.0" psycopg = "^3.1.8" pytest-mock = "^3.10.0" From c3aa59355f2a03077660b85cdea12da644c00c3e Mon Sep 17 00:00:00 2001 From: hkey <81494947+hkey0@users.noreply.github.com> Date: Sun, 31 Dec 2023 04:08:09 +0300 Subject: [PATCH 08/20] Made more readable (pep8) (#10) * made more readable * made more readable * made more readable * made more readable --- src/sinker/sinker.py | 4 ++-- tests/test_bulk_action_generator.py | 10 ++++++---- tests/test_e2e.py | 24 ++++++++++++++++-------- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/sinker/sinker.py b/src/sinker/sinker.py index d8e4a25..656e516 100644 --- a/src/sinker/sinker.py +++ b/src/sinker/sinker.py @@ -25,7 +25,7 @@ BACKFILL_CURSOR_NAME = "backfill" class Sinker: - def __init__(self, view, index): + def __init__(self, view: str, index: str): """ :param view: Postgres materialized view name :param index: Elasticsearch index name @@ -83,7 +83,7 @@ class Sinker: settings=index_body["settings"], ) - def setup_pg(self): + def setup_pg(self) -> None: """ Creates materialized views with supporting functions and triggers, and sets the parent table that drives the materialized view diff --git a/tests/test_bulk_action_generator.py b/tests/test_bulk_action_generator.py index 6ee60a0..d3a5c32 100644 --- a/tests/test_bulk_action_generator.py +++ b/tests/test_bulk_action_generator.py @@ -4,20 +4,22 @@ from sinker.bulk_action_generator import BulkActionGenerator @pytest.fixture -def bulk_action_generator(): +def bulk_action_generator() -> BulkActionGenerator: v2i = dict(foo_mv="foo_index") pt2i = dict(foo_table="foo_index") - return BulkActionGenerator(views_to_indices=v2i, parent_tables_to_indices=pt2i) + return BulkActionGenerator( + views_to_indices=v2i, + parent_tables_to_indices=pt2i) -def test_index_action(bulk_action_generator): +def test_index_action(bulk_action_generator: BulkActionGenerator) -> None: doc = '{"name" : "Foo Bar"}' slot_dict = dict(table="foo_mv", id="a-1") expected = {"_index": "foo_index", "_id": "a-1", "_source": doc} assert bulk_action_generator.index_action(doc, slot_dict) == expected -def test_delete_action(bulk_action_generator): +def test_delete_action(bulk_action_generator: BulkActionGenerator) -> None: slot_dict = dict(table="foo_table", id="a-1") expected = {"_op_type": "delete", "_index": "foo_index", "_id": "a-1"} assert bulk_action_generator.delete_action(slot_dict) == expected diff --git a/tests/test_e2e.py b/tests/test_e2e.py index e753606..f0b5fe7 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -34,20 +34,25 @@ def wait_for_pg_es(): def test_exception_propagation_during_refresh(wait_for_pg_es): runner = Runner() - # Update a record in Postgres but remove the materialized view so it errors when we iterate + # Update a record in Postgres but remove the materialized view so it + # errors when we iterate conn = psycopg.connect(autocommit=True) cur = conn.cursor() cur.execute("update public.person set name = 'Uh oh' where id = 'p-1'") cur.execute("drop materialized view sinker.person_mv") - # make sure the exception gets raised from the thread versus silently consuming it + # make sure the exception gets raised from the thread versus silently + # consuming it with pytest.raises(psycopg.errors.UndefinedTable): runner.iterate() def test_exception_propagation_during_setup(wait_for_pg_es, mocker): - # mock es.get_client() to raise an exception in the thread that sets up the sinker - mocker.patch("sinker.sinker.get_client", side_effect=elasticsearch.exceptions.ConnectionError('Boom!')) - # make sure the exception gets raised from the thread versus silently consuming it + # mock es.get_client() to raise an exception in the thread that sets up + # the sinker + mocker.patch("sinker.sinker.get_client", + side_effect=elasticsearch.exceptions.ConnectionError('Boom!')) + # make sure the exception gets raised from the thread versus silently + # consuming it with pytest.raises(elasticsearch.exceptions.ConnectionError): Runner() @@ -92,12 +97,14 @@ def test_end_to_end(wait_for_pg_es): assert doc["_source"] == expected # Update a record in Postgres - psycopg.connect(autocommit=True).execute("update public.person set name = 'Jane' where id = 'p-1'") + psycopg.connect(autocommit=True).execute( + "update public.person set name = 'Jane' where id = 'p-1'") runner.iterate() # Verify the update made it into the people index doc = es.get(index="people", id="p-1") assert doc["_source"]["name"] == "Jane" - # Verify the update made it into the courses index through person->student->enrollment->course. + # Verify the update made it into the courses index through + # person->student->enrollment->course. doc = es.get(index="courses", id="c-1") expected = { "name": "Reth", @@ -137,7 +144,8 @@ def test_end_to_end(wait_for_pg_es): # Verify the doc got deleted from Elasticsearch with pytest.raises(elasticsearch.NotFoundError): es.get(index="people", id="p-1") - # Verify the doc got deleted from Elasticsearch through person->student->enrollment->course. + # Verify the doc got deleted from Elasticsearch through + # person->student->enrollment->course. doc = es.get(index="courses", id="c-1") expected = { "name": "Reth", From 3165361be01550994cac126b6f502d61d70d810a Mon Sep 17 00:00:00 2001 From: hkey <81494947+hkey0@users.noreply.github.com> Date: Tue, 2 Jan 2024 19:03:19 +0300 Subject: [PATCH 09/20] typos corrected and added some explanations (#8) * typos corrected * fixed as requested --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 14c0258..9fb83d2 100644 --- a/README.md +++ b/README.md @@ -331,7 +331,7 @@ Spin up Postgres and Elasticsearch: docker-compose --env-file=.env.test up -d ``` -Run tests: +Run tests (with -s option to allow more verbose output): ```shell poetry run pytest -s From 59f7af102d9dc0252346e488291f5fd3f28f28e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Jan 2024 10:06:04 -0800 Subject: [PATCH 10/20] Update flake8 requirement from ^6.0.0 to >=6,<8 (#11) Updates the requirements on [flake8](https://github.com/pycqa/flake8) to permit the latest version. - [Commits](https://github.com/pycqa/flake8/compare/6.0.0...7.0.0) --- updated-dependencies: - dependency-name: flake8 dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 613c803..ee07218 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ psycopg = "^3.1.8" pytest-mock = "^3.10.0" [tool.poetry.group.dev.dependencies] -flake8 = "^6.0.0" +flake8 = ">=6,<8" black = "^23.1.0" mypy = "^1.0.0" psycopg-binary = "^3.1.8" From c82c02b7a5cf0d26cc09f507bf407b50bed55d4b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jan 2024 14:18:21 -0800 Subject: [PATCH 11/20] Update pytest requirement from ^7.2.1 to >=7.2.1,<9.0.0 (#12) Updates the requirements on [pytest](https://github.com/pytest-dev/pytest) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.2.1...8.0.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ee07218..f7d45ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ mypy = "^1.0.0" psycopg-binary = "^3.1.8" [tool.poetry.group.test.dependencies] -pytest = "^7.2.1" +pytest = ">=7.2.1,<9.0.0" pytest-cov = "*" [build-system] From c54e37e9b7becf39a452b6f9f18a3b1ed2f2dd16 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 08:53:04 -0700 Subject: [PATCH 12/20] Update environs requirement from >=9.5,<11.0 to >=9.5,<12.0 (#14) Updates the requirements on [environs](https://github.com/sloria/environs) to permit the latest version. - [Changelog](https://github.com/sloria/environs/blob/main/CHANGELOG.md) - [Commits](https://github.com/sloria/environs/compare/9.5.0...11.0.0) --- updated-dependencies: - dependency-name: environs dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f7d45ea..8653171 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ sinker = "sinker.__main__:main" [tool.poetry.dependencies] python = "^3.9" elasticsearch = "^8.6.1" -environs = ">=9.5,<11.0" +environs = ">=9.5,<12.0" psycopg = "^3.1.8" pytest-mock = "^3.10.0" From a7840701ff20f2ef94e74ae3c66416f31e243e75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Mar 2024 09:03:34 -0700 Subject: [PATCH 13/20] Update black requirement from ^23.1.0 to >=23.1,<25.0 (#13) * Update black requirement from ^23.1.0 to >=23.1,<25.0 Updates the requirements on [black](https://github.com/psf/black) to permit the latest version. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/23.1.0...24.1.1) --- updated-dependencies: - dependency-name: black dependency-type: direct:development ... Signed-off-by: dependabot[bot] * new black formatting --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Loren Siebert --- pyproject.toml | 2 +- src/sinker/settings.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8653171..abb4bc5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ pytest-mock = "^3.10.0" [tool.poetry.group.dev.dependencies] flake8 = ">=6,<8" -black = "^23.1.0" +black = ">=23.1,<25.0" mypy = "^1.0.0" psycopg-binary = "^3.1.8" diff --git a/src/sinker/settings.py b/src/sinker/settings.py index 78a172c..212a05b 100644 --- a/src/sinker/settings.py +++ b/src/sinker/settings.py @@ -1,4 +1,5 @@ """Sinker settings.""" + import logging.config import os From e4bf6b89059ad018de9d7b59d7b73442d5ae553a Mon Sep 17 00:00:00 2001 From: Loren Siebert Date: Tue, 14 Jan 2025 13:29:39 -0800 Subject: [PATCH 14/20] Upgrade Elasticsearch from 8.6.1 to 8.17.0 (#18) --- .env.sample | 2 +- .env.test | 2 +- .github/workflows/test.yml | 4 ++-- README.md | 5 +++-- docker-compose.yml | 2 -- pyproject.toml | 4 ++-- src/sinker/runner.py | 4 +++- src/sinker/settings.py | 2 +- src/sinker/sinker.py | 4 +++- 9 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.env.sample b/.env.sample index 3edbe34..ca72eaa 100644 --- a/.env.sample +++ b/.env.sample @@ -21,7 +21,7 @@ ELASTICSEARCH_VERIFY_CERTS=False ELASTICSEARCH_TIMEOUT=600 # Version of Elastic products -STACK_VERSION=8.6.1 +STACK_VERSION=8.17.0 # Set the cluster name CLUSTER_NAME=docker-cluster diff --git a/.env.test b/.env.test index f89b7f7..567bd5b 100644 --- a/.env.test +++ b/.env.test @@ -17,4 +17,4 @@ ELASTICSEARCH_SCHEME=http ELASTICSEARCH_TIMEOUT=600 # Version of Elastic products -STACK_VERSION=8.6.1 +STACK_VERSION=8.17.0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0f5abed..449a89f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,7 @@ jobs: - name: Run image uses: abatilo/actions-poetry@v2 with: - poetry-version: 1.4.0 + poetry-version: 1.8.4 - name: Install dependencies run: poetry install - name: Black @@ -34,7 +34,7 @@ jobs: - name: Copy test envvars run: cp .env.test .env - name: Start docker containers - run: docker-compose up --detach + run: docker compose up --detach - name: Tests run: poetry run pytest --cov=src --cov-report=xml - name: Upload coverage reports to Codecov diff --git a/README.md b/README.md index 9fb83d2..a7f027c 100644 --- a/README.md +++ b/README.md @@ -307,9 +307,9 @@ and [`docker-compose`](https://docs.docker.com/compose/install/) installed. ```shell % docker-compose --version -Docker Compose version v2.15.1 +Docker Compose version v2.31.0 % poetry --version -Poetry (version 1.4.0) +Poetry (version 1.8.4) ``` Clone the repo: @@ -334,6 +334,7 @@ docker-compose --env-file=.env.test up -d Run tests (with -s option to allow more verbose output): ```shell +cp .env.test .env # copy the test environment variables poetry run pytest -s ``` diff --git a/docker-compose.yml b/docker-compose.yml index 50e7bc8..d2c8e8f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.9' - services: postgres: image: postgres:15 diff --git a/pyproject.toml b/pyproject.toml index abb4bc5..8631c44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,8 +11,8 @@ sinker = "sinker.__main__:main" [tool.poetry.dependencies] python = "^3.9" -elasticsearch = "^8.6.1" -environs = ">=9.5,<12.0" +elasticsearch = "^8.17.0" +environs = ">=9.5,<15.0" psycopg = "^3.1.8" pytest-mock = "^3.10.0" diff --git a/src/sinker/runner.py b/src/sinker/runner.py index 617d50b..56b60c4 100644 --- a/src/sinker/runner.py +++ b/src/sinker/runner.py @@ -111,5 +111,7 @@ class Runner: # activity, like inserts into a schema/table you aren't synchronizing to Elasticsearch. If you are worried # about your replication slot growing too large during a scenario like this, you can periodically trigger # a materialized view refresh to clear out the slot (see the CREATE_TODO_ENTRY query template). - processed_tuples, _ = bulk(get_client(), self.bulk_gen.generate_actions(), **ELASTICSEARCH_BULK_KWARGS) + processed_tuples, _ = bulk( + client=get_client(), actions=self.bulk_gen.generate_actions(), stats_only=False, **ELASTICSEARCH_BULK_KWARGS + ) logger.info(f"Processed {processed_tuples} tuples from replication slot") diff --git a/src/sinker/settings.py b/src/sinker/settings.py index 212a05b..b9203ee 100644 --- a/src/sinker/settings.py +++ b/src/sinker/settings.py @@ -13,7 +13,7 @@ SCHEMA_TABLE_DELIMITER = "." # Sinker: # path to the views/indices configs -SINKER_DEFINITIONS_PATH = env.str("SINKER_DEFINITIONS_PATH", default=None) +SINKER_DEFINITIONS_PATH = env.str("SINKER_DEFINITIONS_PATH", default=".") SINKER_SCHEMA = env.str("SINKER_SCHEMA", default=DEFAULT_SCHEMA) SINKER_REPLICATION_SLOT = env.str("SINKER_REPLICATION_SLOT", default="sinker") SINKER_TODO_TABLE = env.str("SINKER_TODO_TABLE", default="todo") diff --git a/src/sinker/sinker.py b/src/sinker/sinker.py index 656e516..a538618 100644 --- a/src/sinker/sinker.py +++ b/src/sinker/sinker.py @@ -52,7 +52,9 @@ class Sinker: def backfill_index(self) -> None: # populate ES index with initial data from materialized view logger.info(f"Populating {self.index} with initial data from {self.view}") - added_docs, _ = bulk(get_client(), self.backfill_stream(), **ELASTICSEARCH_BULK_KWARGS) + added_docs, _ = bulk( + client=get_client(), actions=self.backfill_stream(), stats_only=False, **ELASTICSEARCH_BULK_KWARGS + ) logger.info(f"Added {added_docs} documents to {self.index}") def backfill_stream(self) -> Iterable[Dict[str, Any]]: From 1aaa70f944f3e0dca620289a7ef70a0cc3a8d7e5 Mon Sep 17 00:00:00 2001 From: Loren Siebert Date: Tue, 14 Jan 2025 13:35:26 -0800 Subject: [PATCH 15/20] Support Python 3.13 in test matrix (#19) --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 449a89f..2186167 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12" ] + python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13" ] steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 From 89f2f82685216bf74fab3c14bf3a6f93f6f75bcd Mon Sep 17 00:00:00 2001 From: Loren Siebert Date: Tue, 14 Jan 2025 13:39:56 -0800 Subject: [PATCH 16/20] Support Postgres 17 in tests (#20) --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d2c8e8f..255154b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: postgres: - image: postgres:15 + image: postgres:17 command: postgres -c wal_level=logical -c max_replication_slots=10 environment: POSTGRES_USER: dev From 74db1114dd09b017d3b49bbe0c6800f769bb048f Mon Sep 17 00:00:00 2001 From: Loren Siebert Date: Wed, 15 Jan 2025 16:57:00 -0800 Subject: [PATCH 17/20] Support common table expressions (CTEs) (#21) --- pyproject.toml | 3 +- src/sinker/sinker.py | 26 +++++----- src/sinker/utils.py | 28 +++++------ tests/test_generate_schema_tables.py | 15 ------ tests/test_parse_schema_tables.py | 72 ++++++++++++++++++++++++++++ 5 files changed, 101 insertions(+), 43 deletions(-) delete mode 100644 tests/test_generate_schema_tables.py create mode 100644 tests/test_parse_schema_tables.py diff --git a/pyproject.toml b/pyproject.toml index 8631c44..88a8ff9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "sinker" -version = "0.1.2" +version = "0.1.3" description = "Synchronize Postgres to Elasticsearch" authors = ["Loren Siebert "] license = "MIT/Apache-2.0" @@ -15,6 +15,7 @@ elasticsearch = "^8.17.0" environs = ">=9.5,<15.0" psycopg = "^3.1.8" pytest-mock = "^3.10.0" +sqlglot = "^26.2.1" [tool.poetry.group.dev.dependencies] flake8 = ">=6,<8" diff --git a/src/sinker/sinker.py b/src/sinker/sinker.py index a538618..6047dd6 100644 --- a/src/sinker/sinker.py +++ b/src/sinker/sinker.py @@ -17,7 +17,7 @@ from .settings import ( PGCHUNK_SIZE, SCHEMA_TABLE_DELIMITER, ) -from .utils import generate_schema_tables +from .utils import parse_schema_tables logger = logging.getLogger(__name__) @@ -107,23 +107,23 @@ class Sinker: plpgsql: str = f"{schema_view_name}_fn" create_function: str = q.CREATE_FUNCTION.format(plpgsql, SINKER_SCHEMA, SINKER_TODO_TABLE, schema_view_name) ddl_list.append(create_function) - for schema_table in generate_schema_tables(view_select_query): + # The last table is the top-level table that gets DELETE events with an ID in the replication slot. + # The materialized views do not contain the ID of the doc being deleted, + # so we'll use this table's delete events as a proxy. + # lsn,xid,data + # 0/24EDA4D8,17393,BEGIN 17393 + # 0/24EDA4D8,17393,"table public.""Foo"": DELETE: id[text]:'91754ea9-2983-4cf7-bdf9-fc23d2386d90'" + # 0/24EDC1B0,17393,COMMIT 17393 + # 0/24EDC228,17394,BEGIN 17394 + # 0/24EF0D60,17394,table sinker.foo_mv: DELETE: (no-tuple-data) + # 0/24EF4718,17394,COMMIT 17394 + self.parent_table, schema_tables = parse_schema_tables(view_select_query) + for schema_table in schema_tables: schema, _, table = schema_table.rpartition(SCHEMA_TABLE_DELIMITER) schema = schema or DEFAULT_SCHEMA trigger_name: str = f"{SINKER_SCHEMA}_{self.view}_{schema}_{table}" create_trigger: str = q.CREATE_TRIGGER.format(trigger_name, schema, table, plpgsql) ddl_list.append(create_trigger) - # The last table is the top-level table that gets DELETE events with an ID in the replication slot. - # The materialized views do not contain the ID of the doc being deleted, - # so we'll use this table's delete events as a proxy. - # lsn,xid,data - # 0/24EDA4D8,17393,BEGIN 17393 - # 0/24EDA4D8,17393,"table public.""Foo"": DELETE: id[text]:'91754ea9-2983-4cf7-bdf9-fc23d2386d90'" - # 0/24EDC1B0,17393,COMMIT 17393 - # 0/24EDC228,17394,BEGIN 17394 - # 0/24EF0D60,17394,table sinker.foo_mv: DELETE: (no-tuple-data) - # 0/24EF4718,17394,COMMIT 17394 - self.parent_table = schema_table create_todo_entry: str = q.CREATE_TODO_ENTRY.format(SINKER_SCHEMA, SINKER_TODO_TABLE, schema_view_name) ddl_list.append(create_todo_entry) psycopg.connect(autocommit=True).execute("; ".join(ddl_list)) diff --git a/src/sinker/utils.py b/src/sinker/utils.py index cfed5c9..f8fde3b 100644 --- a/src/sinker/utils.py +++ b/src/sinker/utils.py @@ -1,20 +1,20 @@ -import re +from typing import Set, Tuple -from typing import Iterable - -TABLE_RE = re.compile(r"from\s\"?(\S+)\b", re.I) +import sqlglot +from sqlglot.expressions import Table, CTE -def generate_schema_tables(view_select_query: str) -> Iterable[str]: +def parse_schema_tables(view_select_query: str) -> Tuple[str, Set[str]]: """ - Given a view select query, return a list of unique tables that are referenced in the query - in the order they were encountered. - Skip anything that looks like a function call. + Given a view select query, return a primary parent table and the set of unique tables that are referenced in the + query. Skip anything that looks like a function call. :param view_select_query: The select query from the view """ - seen: set = set() - for table_candidate in TABLE_RE.findall(view_select_query): - if "(" not in table_candidate: - if table_candidate not in seen: - seen.add(table_candidate) - yield table_candidate + parsed = sqlglot.parse_one(view_select_query) + parent_table = parsed.find(Table) + if parent_table is None: + raise ValueError("No table found in the query") + tables = {table.name for table in parsed.find_all(Table)} + ctes = {cte.alias for cte in parsed.find_all(CTE)} + schema_tables = tables - ctes + return parent_table.name, schema_tables diff --git a/tests/test_generate_schema_tables.py b/tests/test_generate_schema_tables.py deleted file mode 100644 index ef8cb14..0000000 --- a/tests/test_generate_schema_tables.py +++ /dev/null @@ -1,15 +0,0 @@ -from sinker.utils import generate_schema_tables - - -def test_generate_schema_tables(): - view_select_query = """select id, - json_build_object( - 'name', "name", - 'otherEmailDomains',(select array_agg(split_part(email, '@', 2)) FROM unnest(emails) as email), - 'emailDomains', (select array_agg(split_part(value, '@', 2)) - from "EmailAddress" EA where "personId"="Person".id), - 'emailAddresses', (select array_agg(value) from "EmailAddress" EA where "personId"="Person".id), - ) as "person" - from "person" - """ - assert list(generate_schema_tables(view_select_query)) == ["EmailAddress", "person"] diff --git a/tests/test_parse_schema_tables.py b/tests/test_parse_schema_tables.py new file mode 100644 index 0000000..a47e75f --- /dev/null +++ b/tests/test_parse_schema_tables.py @@ -0,0 +1,72 @@ +from sinker.utils import parse_schema_tables + + +def test_parse_schema_tables(): + view_select_query = """select id, + json_build_object( + 'name', "name", + 'otherEmailDomains',(select array_agg(split_part(email, '@', 2)) FROM unnest(emails) as email), + 'emailDomains', (select array_agg(split_part(value, '@', 2)) + from "EmailAddress" EA where "personId"="Person".id), + 'emailAddresses', (select array_agg(value) from "EmailAddress" EA where "personId"="Person".id), + ) as "person" + from "person" + """ + parent_table, schema_tables = parse_schema_tables(view_select_query) + assert parent_table == "person" + assert schema_tables == {"EmailAddress", "person"} + + +def test_parse_schema_tables_with_cte(): + view_select_query = """ + WITH + attendees AS ( + SELECT DISTINCT ON (a."personId", a."hostedEventId") + a."hostedEventId", + a.status, + e.value as email, + p."primaryOrganizationId" + FROM "HostedEventAttendance" a + JOIN "Person" p ON a."personId" = p.id + JOIN "EmailAddress" e ON p.id = e."personId" + GROUP BY + a."personId", + a."hostedEventId", + a.status, + e.value, + p."primaryOrganizationId" + ) + SELECT + id, + json_build_object( + 'summary', "name", + 'startTime', "timestamp", + 'attendees', ( + SELECT json_agg(json_build_object('email', attendees.email, 'eventResponse', attendees.status)) + AS formatted_attendees + FROM attendees + WHERE attendees."hostedEventId" = "HostedEvent".id + ), + 'organizationIds', + ( + SELECT array_agg(attendees."primaryOrganizationId") + FROM attendees + WHERE attendees."hostedEventId" = "HostedEvent".id + ) + ) AS "hosted_events" + FROM + "HostedEvent" + """ + parent_table, schema_tables = parse_schema_tables(view_select_query) + assert parent_table == "HostedEvent" + assert schema_tables == {"EmailAddress", "HostedEvent", "HostedEventAttendance", "Person"} + + +def test_error_handling_on_query_with_no_table(): + view_select_query = """select 1""" + try: + parse_schema_tables(view_select_query) + except ValueError as e: + assert str(e) == "No table found in the query" + else: + assert False, "Expected ValueError" From be9692e3f7a58c093614a710d1fd22b99bdd22cd Mon Sep 17 00:00:00 2001 From: Loren Siebert Date: Thu, 16 Jan 2025 11:42:05 -0800 Subject: [PATCH 18/20] Fixes regression in parse_schema_tables (#22) --- src/sinker/utils.py | 2 +- tests/test_parse_schema_tables.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/sinker/utils.py b/src/sinker/utils.py index f8fde3b..c84dbe3 100644 --- a/src/sinker/utils.py +++ b/src/sinker/utils.py @@ -14,7 +14,7 @@ def parse_schema_tables(view_select_query: str) -> Tuple[str, Set[str]]: parent_table = parsed.find(Table) if parent_table is None: raise ValueError("No table found in the query") - tables = {table.name for table in parsed.find_all(Table)} + tables = {table.name for table in parsed.find_all(Table) if table.name} ctes = {cte.alias for cte in parsed.find_all(CTE)} schema_tables = tables - ctes return parent_table.name, schema_tables diff --git a/tests/test_parse_schema_tables.py b/tests/test_parse_schema_tables.py index a47e75f..559ce06 100644 --- a/tests/test_parse_schema_tables.py +++ b/tests/test_parse_schema_tables.py @@ -62,6 +62,32 @@ def test_parse_schema_tables_with_cte(): assert schema_tables == {"EmailAddress", "HostedEvent", "HostedEventAttendance", "Person"} +def test_parse_schema_tables_handles_blank_table_name(): + view_select_query = """ + select id, + json_build_object( + 'summary', "summary", + 'startTime', "start_time", + 'organizerEmail', "organizerEmail", + 'attendees', (select json_agg(json_build_object('email', key, 'eventResponse', value)) + as formatted_attendees + from (select id, key, value + from "googleEvents", + jsonb_each_text(attendees) as kv(key, value)) as subquery + where id = "googleEvents".id), + 'organizationIds', (select array_agg("_NotesToOrganization"."B") + from "_NotesToOrganization" + left join public."Notes" N on "_NotesToOrganization"."A" = N.id + where "googleEventId" = "googleEvents".id) + ) as "google_events" + from "googleEvents"; + """ + parent_table, schema_tables = parse_schema_tables(view_select_query) + assert parent_table == "googleEvents" + # parsed.find_all(Table) returns a Table object with a blank '' name + assert schema_tables == {"googleEvents", "_NotesToOrganization", "Notes"} + + def test_error_handling_on_query_with_no_table(): view_select_query = """select 1""" try: From 48e8c0494faad63a1e8062d22866f1a0297db116 Mon Sep 17 00:00:00 2001 From: Loren Siebert Date: Thu, 16 Jan 2025 12:06:32 -0800 Subject: [PATCH 19/20] Bump version to 0.1.4 (#23) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 88a8ff9..020cde7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "sinker" -version = "0.1.3" +version = "0.1.4" description = "Synchronize Postgres to Elasticsearch" authors = ["Loren Siebert "] license = "MIT/Apache-2.0" From fbef5c2fa48daaf5b97e1b701441719aef63d516 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2025 08:21:29 -0800 Subject: [PATCH 20/20] Update black requirement from >=23.1,<25.0 to >=23.1,<26.0 (#24) Updates the requirements on [black](https://github.com/psf/black) to permit the latest version. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/23.1.0...25.1.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 020cde7..a966cf5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ sqlglot = "^26.2.1" [tool.poetry.group.dev.dependencies] flake8 = ">=6,<8" -black = ">=23.1,<25.0" +black = ">=23.1,<26.0" mypy = "^1.0.0" psycopg-binary = "^3.1.8"