diff --git a/.env.sample b/.env.sample index ca72eaa..3edbe34 100644 --- a/.env.sample +++ b/.env.sample @@ -21,7 +21,7 @@ ELASTICSEARCH_VERIFY_CERTS=False ELASTICSEARCH_TIMEOUT=600 # Version of Elastic products -STACK_VERSION=8.17.0 +STACK_VERSION=8.6.1 # Set the cluster name CLUSTER_NAME=docker-cluster diff --git a/.env.test b/.env.test index 567bd5b..f89b7f7 100644 --- a/.env.test +++ b/.env.test @@ -17,4 +17,4 @@ ELASTICSEARCH_SCHEME=http ELASTICSEARCH_TIMEOUT=600 # Version of Elastic products -STACK_VERSION=8.17.0 +STACK_VERSION=8.6.1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2186167..32a41de 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13" ] + python-version: [ "3.9", "3.10", "3.11" ] steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -22,7 +22,7 @@ jobs: - name: Run image uses: abatilo/actions-poetry@v2 with: - poetry-version: 1.8.4 + poetry-version: 1.4.0 - name: Install dependencies run: poetry install - name: Black @@ -34,10 +34,10 @@ jobs: - name: Copy test envvars run: cp .env.test .env - name: Start docker containers - run: docker compose up --detach + run: docker-compose up --detach - name: Tests run: poetry run pytest --cov=src --cov-report=xml - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 with: - token: ${{ secrets.CODECOV_TOKEN }} + token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file diff --git a/README.md b/README.md index a7f027c..14c0258 100644 --- a/README.md +++ b/README.md @@ -307,9 +307,9 @@ and [`docker-compose`](https://docs.docker.com/compose/install/) installed. ```shell % docker-compose --version -Docker Compose version v2.31.0 +Docker Compose version v2.15.1 % poetry --version -Poetry (version 1.8.4) +Poetry (version 1.4.0) ``` Clone the repo: @@ -331,10 +331,9 @@ Spin up Postgres and Elasticsearch: docker-compose --env-file=.env.test up -d ``` -Run tests (with -s option to allow more verbose output): +Run tests: ```shell -cp .env.test .env # copy the test environment variables poetry run pytest -s ``` diff --git a/docker-compose.yml b/docker-compose.yml index 255154b..50e7bc8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,8 @@ +version: '3.9' + services: postgres: - image: postgres:17 + image: postgres:15 command: postgres -c wal_level=logical -c max_replication_slots=10 environment: POSTGRES_USER: dev diff --git a/pyproject.toml b/pyproject.toml index a966cf5..495a369 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "sinker" -version = "0.1.4" +version = "0.1.0" description = "Synchronize Postgres to Elasticsearch" authors = ["Loren Siebert "] license = "MIT/Apache-2.0" @@ -11,20 +11,19 @@ sinker = "sinker.__main__:main" [tool.poetry.dependencies] python = "^3.9" -elasticsearch = "^8.17.0" -environs = ">=9.5,<15.0" +elasticsearch = "^8.6.1" +environs = "^9.5.0" psycopg = "^3.1.8" pytest-mock = "^3.10.0" -sqlglot = "^26.2.1" [tool.poetry.group.dev.dependencies] -flake8 = ">=6,<8" -black = ">=23.1,<26.0" +flake8 = "^6.0.0" +black = "^23.1.0" mypy = "^1.0.0" psycopg-binary = "^3.1.8" [tool.poetry.group.test.dependencies] -pytest = ">=7.2.1,<9.0.0" +pytest = "^7.2.1" pytest-cov = "*" [build-system] diff --git a/src/sinker/runner.py b/src/sinker/runner.py index 56b60c4..617d50b 100644 --- a/src/sinker/runner.py +++ b/src/sinker/runner.py @@ -111,7 +111,5 @@ class Runner: # activity, like inserts into a schema/table you aren't synchronizing to Elasticsearch. If you are worried # about your replication slot growing too large during a scenario like this, you can periodically trigger # a materialized view refresh to clear out the slot (see the CREATE_TODO_ENTRY query template). - processed_tuples, _ = bulk( - client=get_client(), actions=self.bulk_gen.generate_actions(), stats_only=False, **ELASTICSEARCH_BULK_KWARGS - ) + processed_tuples, _ = bulk(get_client(), self.bulk_gen.generate_actions(), **ELASTICSEARCH_BULK_KWARGS) logger.info(f"Processed {processed_tuples} tuples from replication slot") diff --git a/src/sinker/settings.py b/src/sinker/settings.py index b9203ee..78a172c 100644 --- a/src/sinker/settings.py +++ b/src/sinker/settings.py @@ -1,5 +1,4 @@ """Sinker settings.""" - import logging.config import os @@ -13,7 +12,7 @@ SCHEMA_TABLE_DELIMITER = "." # Sinker: # path to the views/indices configs -SINKER_DEFINITIONS_PATH = env.str("SINKER_DEFINITIONS_PATH", default=".") +SINKER_DEFINITIONS_PATH = env.str("SINKER_DEFINITIONS_PATH", default=None) SINKER_SCHEMA = env.str("SINKER_SCHEMA", default=DEFAULT_SCHEMA) SINKER_REPLICATION_SLOT = env.str("SINKER_REPLICATION_SLOT", default="sinker") SINKER_TODO_TABLE = env.str("SINKER_TODO_TABLE", default="todo") diff --git a/src/sinker/sinker.py b/src/sinker/sinker.py index 6047dd6..70b1ba6 100644 --- a/src/sinker/sinker.py +++ b/src/sinker/sinker.py @@ -1,6 +1,7 @@ import json import logging import os +import re from typing import Iterable, Dict, Any import psycopg @@ -17,15 +18,15 @@ from .settings import ( PGCHUNK_SIZE, SCHEMA_TABLE_DELIMITER, ) -from .utils import parse_schema_tables logger = logging.getLogger(__name__) BACKFILL_CURSOR_NAME = "backfill" +TABLE_RE = re.compile(r"from\s\"?(\S+)\b", re.I) class Sinker: - def __init__(self, view: str, index: str): + def __init__(self, view, index): """ :param view: Postgres materialized view name :param index: Elasticsearch index name @@ -52,9 +53,7 @@ class Sinker: def backfill_index(self) -> None: # populate ES index with initial data from materialized view logger.info(f"Populating {self.index} with initial data from {self.view}") - added_docs, _ = bulk( - client=get_client(), actions=self.backfill_stream(), stats_only=False, **ELASTICSEARCH_BULK_KWARGS - ) + added_docs, _ = bulk(get_client(), self.backfill_stream(), **ELASTICSEARCH_BULK_KWARGS) logger.info(f"Added {added_docs} documents to {self.index}") def backfill_stream(self) -> Iterable[Dict[str, Any]]: @@ -85,7 +84,7 @@ class Sinker: settings=index_body["settings"], ) - def setup_pg(self) -> None: + def setup_pg(self): """ Creates materialized views with supporting functions and triggers, and sets the parent table that drives the materialized view @@ -107,6 +106,16 @@ class Sinker: plpgsql: str = f"{schema_view_name}_fn" create_function: str = q.CREATE_FUNCTION.format(plpgsql, SINKER_SCHEMA, SINKER_TODO_TABLE, schema_view_name) ddl_list.append(create_function) + schema_tables: list[Any] = TABLE_RE.findall(view_select_query) + for schema_table in schema_tables: + schema, _, table = schema_table.rpartition(SCHEMA_TABLE_DELIMITER) + schema = schema or DEFAULT_SCHEMA + trigger_name: str = f"{SINKER_SCHEMA}_{self.view}_{schema}_{table}" + create_trigger: str = q.CREATE_TRIGGER.format(trigger_name, schema, table, plpgsql) + ddl_list.append(create_trigger) + create_todo_entry: str = q.CREATE_TODO_ENTRY.format(SINKER_SCHEMA, SINKER_TODO_TABLE, schema_view_name) + ddl_list.append(create_todo_entry) + psycopg.connect(autocommit=True).execute("; ".join(ddl_list)) # The last table is the top-level table that gets DELETE events with an ID in the replication slot. # The materialized views do not contain the ID of the doc being deleted, # so we'll use this table's delete events as a proxy. @@ -117,16 +126,7 @@ class Sinker: # 0/24EDC228,17394,BEGIN 17394 # 0/24EF0D60,17394,table sinker.foo_mv: DELETE: (no-tuple-data) # 0/24EF4718,17394,COMMIT 17394 - self.parent_table, schema_tables = parse_schema_tables(view_select_query) - for schema_table in schema_tables: - schema, _, table = schema_table.rpartition(SCHEMA_TABLE_DELIMITER) - schema = schema or DEFAULT_SCHEMA - trigger_name: str = f"{SINKER_SCHEMA}_{self.view}_{schema}_{table}" - create_trigger: str = q.CREATE_TRIGGER.format(trigger_name, schema, table, plpgsql) - ddl_list.append(create_trigger) - create_todo_entry: str = q.CREATE_TODO_ENTRY.format(SINKER_SCHEMA, SINKER_TODO_TABLE, schema_view_name) - ddl_list.append(create_todo_entry) - psycopg.connect(autocommit=True).execute("; ".join(ddl_list)) + self.parent_table = schema_tables[-1] def refresh_view(self) -> str: logger.info(f"Refreshing the {self.view} materialized view") diff --git a/src/sinker/utils.py b/src/sinker/utils.py deleted file mode 100644 index c84dbe3..0000000 --- a/src/sinker/utils.py +++ /dev/null @@ -1,20 +0,0 @@ -from typing import Set, Tuple - -import sqlglot -from sqlglot.expressions import Table, CTE - - -def parse_schema_tables(view_select_query: str) -> Tuple[str, Set[str]]: - """ - Given a view select query, return a primary parent table and the set of unique tables that are referenced in the - query. Skip anything that looks like a function call. - :param view_select_query: The select query from the view - """ - parsed = sqlglot.parse_one(view_select_query) - parent_table = parsed.find(Table) - if parent_table is None: - raise ValueError("No table found in the query") - tables = {table.name for table in parsed.find_all(Table) if table.name} - ctes = {cte.alias for cte in parsed.find_all(CTE)} - schema_tables = tables - ctes - return parent_table.name, schema_tables diff --git a/tests/test_bulk_action_generator.py b/tests/test_bulk_action_generator.py index d3a5c32..6ee60a0 100644 --- a/tests/test_bulk_action_generator.py +++ b/tests/test_bulk_action_generator.py @@ -4,22 +4,20 @@ from sinker.bulk_action_generator import BulkActionGenerator @pytest.fixture -def bulk_action_generator() -> BulkActionGenerator: +def bulk_action_generator(): v2i = dict(foo_mv="foo_index") pt2i = dict(foo_table="foo_index") - return BulkActionGenerator( - views_to_indices=v2i, - parent_tables_to_indices=pt2i) + return BulkActionGenerator(views_to_indices=v2i, parent_tables_to_indices=pt2i) -def test_index_action(bulk_action_generator: BulkActionGenerator) -> None: +def test_index_action(bulk_action_generator): doc = '{"name" : "Foo Bar"}' slot_dict = dict(table="foo_mv", id="a-1") expected = {"_index": "foo_index", "_id": "a-1", "_source": doc} assert bulk_action_generator.index_action(doc, slot_dict) == expected -def test_delete_action(bulk_action_generator: BulkActionGenerator) -> None: +def test_delete_action(bulk_action_generator): slot_dict = dict(table="foo_table", id="a-1") expected = {"_op_type": "delete", "_index": "foo_index", "_id": "a-1"} assert bulk_action_generator.delete_action(slot_dict) == expected diff --git a/tests/test_e2e.py b/tests/test_e2e.py index f0b5fe7..e753606 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -34,25 +34,20 @@ def wait_for_pg_es(): def test_exception_propagation_during_refresh(wait_for_pg_es): runner = Runner() - # Update a record in Postgres but remove the materialized view so it - # errors when we iterate + # Update a record in Postgres but remove the materialized view so it errors when we iterate conn = psycopg.connect(autocommit=True) cur = conn.cursor() cur.execute("update public.person set name = 'Uh oh' where id = 'p-1'") cur.execute("drop materialized view sinker.person_mv") - # make sure the exception gets raised from the thread versus silently - # consuming it + # make sure the exception gets raised from the thread versus silently consuming it with pytest.raises(psycopg.errors.UndefinedTable): runner.iterate() def test_exception_propagation_during_setup(wait_for_pg_es, mocker): - # mock es.get_client() to raise an exception in the thread that sets up - # the sinker - mocker.patch("sinker.sinker.get_client", - side_effect=elasticsearch.exceptions.ConnectionError('Boom!')) - # make sure the exception gets raised from the thread versus silently - # consuming it + # mock es.get_client() to raise an exception in the thread that sets up the sinker + mocker.patch("sinker.sinker.get_client", side_effect=elasticsearch.exceptions.ConnectionError('Boom!')) + # make sure the exception gets raised from the thread versus silently consuming it with pytest.raises(elasticsearch.exceptions.ConnectionError): Runner() @@ -97,14 +92,12 @@ def test_end_to_end(wait_for_pg_es): assert doc["_source"] == expected # Update a record in Postgres - psycopg.connect(autocommit=True).execute( - "update public.person set name = 'Jane' where id = 'p-1'") + psycopg.connect(autocommit=True).execute("update public.person set name = 'Jane' where id = 'p-1'") runner.iterate() # Verify the update made it into the people index doc = es.get(index="people", id="p-1") assert doc["_source"]["name"] == "Jane" - # Verify the update made it into the courses index through - # person->student->enrollment->course. + # Verify the update made it into the courses index through person->student->enrollment->course. doc = es.get(index="courses", id="c-1") expected = { "name": "Reth", @@ -144,8 +137,7 @@ def test_end_to_end(wait_for_pg_es): # Verify the doc got deleted from Elasticsearch with pytest.raises(elasticsearch.NotFoundError): es.get(index="people", id="p-1") - # Verify the doc got deleted from Elasticsearch through - # person->student->enrollment->course. + # Verify the doc got deleted from Elasticsearch through person->student->enrollment->course. doc = es.get(index="courses", id="c-1") expected = { "name": "Reth", diff --git a/tests/test_parse_schema_tables.py b/tests/test_parse_schema_tables.py deleted file mode 100644 index 559ce06..0000000 --- a/tests/test_parse_schema_tables.py +++ /dev/null @@ -1,98 +0,0 @@ -from sinker.utils import parse_schema_tables - - -def test_parse_schema_tables(): - view_select_query = """select id, - json_build_object( - 'name', "name", - 'otherEmailDomains',(select array_agg(split_part(email, '@', 2)) FROM unnest(emails) as email), - 'emailDomains', (select array_agg(split_part(value, '@', 2)) - from "EmailAddress" EA where "personId"="Person".id), - 'emailAddresses', (select array_agg(value) from "EmailAddress" EA where "personId"="Person".id), - ) as "person" - from "person" - """ - parent_table, schema_tables = parse_schema_tables(view_select_query) - assert parent_table == "person" - assert schema_tables == {"EmailAddress", "person"} - - -def test_parse_schema_tables_with_cte(): - view_select_query = """ - WITH - attendees AS ( - SELECT DISTINCT ON (a."personId", a."hostedEventId") - a."hostedEventId", - a.status, - e.value as email, - p."primaryOrganizationId" - FROM "HostedEventAttendance" a - JOIN "Person" p ON a."personId" = p.id - JOIN "EmailAddress" e ON p.id = e."personId" - GROUP BY - a."personId", - a."hostedEventId", - a.status, - e.value, - p."primaryOrganizationId" - ) - SELECT - id, - json_build_object( - 'summary', "name", - 'startTime', "timestamp", - 'attendees', ( - SELECT json_agg(json_build_object('email', attendees.email, 'eventResponse', attendees.status)) - AS formatted_attendees - FROM attendees - WHERE attendees."hostedEventId" = "HostedEvent".id - ), - 'organizationIds', - ( - SELECT array_agg(attendees."primaryOrganizationId") - FROM attendees - WHERE attendees."hostedEventId" = "HostedEvent".id - ) - ) AS "hosted_events" - FROM - "HostedEvent" - """ - parent_table, schema_tables = parse_schema_tables(view_select_query) - assert parent_table == "HostedEvent" - assert schema_tables == {"EmailAddress", "HostedEvent", "HostedEventAttendance", "Person"} - - -def test_parse_schema_tables_handles_blank_table_name(): - view_select_query = """ - select id, - json_build_object( - 'summary', "summary", - 'startTime', "start_time", - 'organizerEmail', "organizerEmail", - 'attendees', (select json_agg(json_build_object('email', key, 'eventResponse', value)) - as formatted_attendees - from (select id, key, value - from "googleEvents", - jsonb_each_text(attendees) as kv(key, value)) as subquery - where id = "googleEvents".id), - 'organizationIds', (select array_agg("_NotesToOrganization"."B") - from "_NotesToOrganization" - left join public."Notes" N on "_NotesToOrganization"."A" = N.id - where "googleEventId" = "googleEvents".id) - ) as "google_events" - from "googleEvents"; - """ - parent_table, schema_tables = parse_schema_tables(view_select_query) - assert parent_table == "googleEvents" - # parsed.find_all(Table) returns a Table object with a blank '' name - assert schema_tables == {"googleEvents", "_NotesToOrganization", "Notes"} - - -def test_error_handling_on_query_with_no_table(): - view_select_query = """select 1""" - try: - parse_schema_tables(view_select_query) - except ValueError as e: - assert str(e) == "No table found in the query" - else: - assert False, "Expected ValueError"