diff --git a/crates/python/README.md b/crates/python/README.md index 7807a57..95f1ff2 100644 --- a/crates/python/README.md +++ b/crates/python/README.md @@ -3,3 +3,40 @@ see full [README](https://github.com/paradigmxyz/cryo?tab=readme-ov-file#installing-cryo_python-from-pypi) for details +The `cryo` python package provides the following functions that each take similar arguments: +`cryo.collect(**query)`: extract data over rpc and return as dataframe +`cryo.freeze(**query)`: extract data over rpc and save to disk +`cryo.is_data_saved(**query)`: return whether given data query has been saved to disk +`cryo.scan(**query)`: return `LazyFrame` scan of extracted data from disk +`cryo.get(**query)`: return `LazyFrame` of data from memory, disk, or rpc + +The `query` uses the same parameters as the `cryo` cli. + + +## CRYO_ROOT +- `CRYO_ROOT` is a special directory +- [question] should this be the default or should it be an option like `--core` +- if `CRYO_ROOT` is not set, then the default value of `--output-dir` is `$PWD` +- if `CRYO_ROOT` is set, then the default value of `--output-dir` is + - `$CRYO_ROOT/{NETWORK}/{DATATYPE}` if no special options are used + - `$CRYO_ROOT/{NETWORK}/{DATATYPE}_{LABEL}` if label is specified + - `$CRYO_ROOT/{NETWORK}/{DATATYPE}_{HASH(OPTIONS)}` if Content or Output options are used + + +## CRYO_ROOT layout +- `$CRYO_ROOT/data/{NETWORK}/{DATATYPE[_LABEL]}`: data stored directly in cryo root +- `$CRYO_ROOT/data_links/{NETWORK}/{DATATYPE[_LABEL]}/{timestamp}.link`: symlinks to data stored elsewhere on computer +- `$CRYO_ROOT/metadata` + + + +changes to make +- change --network-name to just --network + - use mesc to convert into chain id + - if --rpc and --network specified, make sure they match +- add `--root` arg to use CRYO_ROOT as output dir + - for now, refuse if any Content or Output options used, need to hash those args +- add is_data_saved() + + + diff --git a/crates/python/pyproject.toml b/crates/python/pyproject.toml index 2d2ce6c..1c5aeaf 100644 --- a/crates/python/pyproject.toml +++ b/crates/python/pyproject.toml @@ -23,5 +23,11 @@ features = ["pyo3/extension-module"] [tool.mypy] strict = true files = ["python/cryo"] -enable_incomplete_feature = "Unpack" + +[tool.ruff.format] +quote-style = "single" + +[tool.ruff.lint.flake8-quotes] +docstring-quotes = "double" +inline-quotes = "single" diff --git a/crates/python/python/cryo/__init__.py b/crates/python/python/cryo/__init__.py index ac472a4..cb322af 100644 --- a/crates/python/python/cryo/__init__.py +++ b/crates/python/python/cryo/__init__.py @@ -2,9 +2,24 @@ # ruff: noqa: F401 -from ._cryo_rust import __version__ +from ._cryo_rust import __version__ # type: ignore from ._freeze import async_freeze +from ._get import get +from ._filesystem import get_glob, get_file_list, is_data_saved, scan from ._freeze import freeze from ._collect import async_collect from ._collect import collect + +__all__ = ( + '__version__', + 'async_collect', + 'async_freeze', + 'collect', + 'freeze', + 'is_data_saved', + 'get', + 'get_glob', + 'get_file_list', + 'scan', +) diff --git a/crates/python/python/cryo/_args.py b/crates/python/python/cryo/_args.py index 01ef0e8..c132fa0 100644 --- a/crates/python/python/cryo/_args.py +++ b/crates/python/python/cryo/_args.py @@ -19,12 +19,7 @@ def parse_cli_args( TODO: write this part in rust """ - if start_block is not None and end_block is not None: - kwargs['blocks'] = [str(start_block) + ':' + str(end_block)] - elif start_block is not None: - kwargs['blocks'] = [str(start_block) + ':'] - elif end_block is not None: - kwargs['blocks'] = [':' + str(end_block)] + kwargs['blocks'] = [get_blocks_str(start_block, end_block)] if file_format == 'parquet': pass @@ -32,8 +27,6 @@ def parse_cli_args( kwargs['json'] = True elif file_format == 'csv': kwargs['csv'] = True - # elif file_format == 'avro': - # kwargs['avro'] = True else: raise Exception('unknown file_format') @@ -41,3 +34,21 @@ def parse_cli_args( return kwargs + +def get_blocks_str( + start_block: int | str | None, + end_block: int | str | None, +) -> str: + if start_block is not None and end_block is not None: + return str(start_block) + ':' + str(end_block) + elif start_block is not None: + return str(start_block) + ':' + elif end_block is not None: + return ':' + str(end_block) + else: + return ':' + + +def parse_blocks_str(blocks_str: str) -> tuple[int, int]: + start_block, end_block = blocks_str.split(':') + return int(start_block), int(end_block) diff --git a/crates/python/python/cryo/_collect.py b/crates/python/python/cryo/_collect.py index 55d7488..a2e0a60 100644 --- a/crates/python/python/cryo/_collect.py +++ b/crates/python/python/cryo/_collect.py @@ -3,7 +3,7 @@ import typing if typing.TYPE_CHECKING: from typing_extensions import Unpack - from typing import Any + from typing import Any, Literal, overload import pandas as pd import polars as pl @@ -13,15 +13,51 @@ if typing.TYPE_CHECKING: DictOfLists = dict[str, list[Any]] +@overload async def async_collect( - datatype: _spec.Datatype, + datatype: _spec.Datatype | None = None, + *, + output_format: Literal['polars'], + **kwargs: Unpack[_spec.CryoCliArgs], +) -> pl.DataFrame: ... + + +@overload +async def async_collect( + datatype: _spec.Datatype | None = None, + *, + output_format: Literal['pandas'], + **kwargs: Unpack[_spec.CryoCliArgs], +) -> pd.DataFrame: ... + + +@overload +async def async_collect( + datatype: _spec.Datatype | None = None, + *, + output_format: Literal['list'], + **kwargs: Unpack[_spec.CryoCliArgs], +) -> ListOfDicts: ... + + +@overload +async def async_collect( + datatype: _spec.Datatype | None = None, + *, + output_format: Literal['dict'], + **kwargs: Unpack[_spec.CryoCliArgs], +) -> DictOfLists: ... + + +async def async_collect( + datatype: _spec.Datatype | None = None, output_format: _spec.PythonOutput = 'polars', **kwargs: Unpack[_spec.CryoCliArgs], ) -> pl.DataFrame | pd.DataFrame | ListOfDicts | DictOfLists: """asynchronously collect data and return as dataframe""" from . import _args - from . import _cryo_rust # type: ignore + from . import _cryo_rust # parse inputs cli_args = _args.parse_cli_args(**kwargs) @@ -45,8 +81,45 @@ async def async_collect( raise Exception('unknown output format') +@overload def collect( - datatype: _spec.Datatype, + datatype: _spec.Datatype | None = None, + *, + output_format: Literal['polars'], + **kwargs: Unpack[_spec.CryoCliArgs], +) -> pl.DataFrame: ... + + +@overload +def collect( + datatype: _spec.Datatype | None = None, + *, + output_format: Literal['pandas'], + **kwargs: Unpack[_spec.CryoCliArgs], +) -> pd.DataFrame: ... + + +@overload +def collect( + datatype: _spec.Datatype | None = None, + *, + output_format: Literal['list'], + **kwargs: Unpack[_spec.CryoCliArgs], +) -> ListOfDicts: ... + + +@overload +def collect( + datatype: _spec.Datatype | None = None, + *, + output_format: Literal['dict'], + **kwargs: Unpack[_spec.CryoCliArgs], +) -> DictOfLists: ... + + +def collect( + datatype: _spec.Datatype | None = None, + *, output_format: _spec.PythonOutput = 'polars', **kwargs: Unpack[_spec.CryoCliArgs], ) -> pl.DataFrame | pd.DataFrame | ListOfDicts | DictOfLists: @@ -58,11 +131,11 @@ def collect( try: import concurrent.futures + loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) with concurrent.futures.ThreadPoolExecutor() as executor: - future = executor.submit(loop.run_until_complete, coroutine) # type: ignore + future = executor.submit(loop.run_until_complete, coroutine) return future.result() # type: ignore except RuntimeError: - return asyncio.run(coroutine) - + return asyncio.run(coroutine) # type: ignore diff --git a/crates/python/python/cryo/_filesystem.py b/crates/python/python/cryo/_filesystem.py new file mode 100644 index 0000000..80e02ac --- /dev/null +++ b/crates/python/python/cryo/_filesystem.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import os +import typing + +import polars as pl + +from . import _filesystem +from . import _spec + +if typing.TYPE_CHECKING: + from typing_extensions import Unpack + + +def get_glob( + datatype: str, + *, + network: str | int, + label: str | None = None, + cryo_root: str | None = None, + data_dir: str | None = None, + start_block: int | None = None, + end_block: int | None = None, + filetype: str = 'parquet', +) -> str: + import mesc + + # get network name + if isinstance(network, str): + network_name = network + else: + network_name_option = mesc.get_network_name(network) + if network_name_option is None: + raise Exception('network name not found') + else: + network_name = network_name_option + + # create filename glob + if label is None: + file_template = '{network}__{datatype}__*.{filetype}' + else: + file_template = '{network}__{datatype}__{label}__*.{filetype}' + file_glob = file_template.format( + network=network_name, + datatype=datatype, + filetype=filetype, + ) + + # get data directory + if data_dir is None: + if cryo_root is not None: + data_dir = os.path.join(cryo_root, network_name) + else: + raise Exception('must specify cryo_root or data_dir') + + # TODO: tune glob string to block range + + return os.path.join(data_dir, file_glob) + + +def get_file_list( + datatype: str, + *, + network: str | int, + label: str | None = None, + cryo_root: str | None = None, + data_dir: str | None = None, + start_block: int | None = None, + end_block: int | None = None, + filetype: str = 'parquet', +) -> list[str]: + # TODO: get more specific + import glob + + path_glob = get_glob( + datatype=datatype, + network=network, + cryo_root=cryo_root, + data_dir=data_dir, + start_block=start_block, + end_block=end_block, + filetype=filetype, + ) + return glob.glob(path_glob) + + +def scan( + datatype: str, + *, + network: str | int, + cryo_root: str | None = None, + data_dir: str | None = None, + start_block: int | None = None, + end_block: int | None = None, + filetype: str = 'parquet', + **kwargs: Unpack[_spec.CryoCliArgs], +) -> pl.LazyFrame: + if len(kwargs) > 0: + raise Exception('args not accepted for scan(): ' + str(list(kwargs.keys()))) + files = _filesystem.get_file_list( + datatype=datatype, + network=network, + cryo_root=cryo_root, + data_dir=data_dir, + start_block=start_block, + end_block=end_block, + filetype=filetype, + ) + return pl.scan_parquet(files) + + +def is_data_saved(datatype: str, **kwargs: Unpack[_spec.CryoCliArgs]) -> bool: + raise NotImplementedError() diff --git a/crates/python/python/cryo/_freeze.py b/crates/python/python/cryo/_freeze.py index f5dd977..d2b14f7 100644 --- a/crates/python/python/cryo/_freeze.py +++ b/crates/python/python/cryo/_freeze.py @@ -17,7 +17,7 @@ async def async_freeze( see cryo.parse_kwargs() for descriptions of arguments """ - from . import _cryo_rust # type: ignore + from . import _cryo_rust from . import _args if isinstance(datatype, str): @@ -47,8 +47,7 @@ def freeze( loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) with concurrent.futures.ThreadPoolExecutor() as executor: - future = executor.submit(loop.run_until_complete, coroutine) # type: ignore - return future.result() # type: ignore + future = executor.submit(loop.run_until_complete, coroutine) + return future.result() except RuntimeError: return asyncio.run(coroutine) - diff --git a/crates/python/python/cryo/_get.py b/crates/python/python/cryo/_get.py new file mode 100644 index 0000000..3d9747a --- /dev/null +++ b/crates/python/python/cryo/_get.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import typing + +import polars as pl + +import cryo +from . import _args +from . import _filesystem +from . import _spec + +if typing.TYPE_CHECKING: + from typing import Mapping, Sequence + from typing_extensions import Unpack + + +def get( + datatype: str, + *, + lfs: Mapping[str, pl.LazyFrame] | None = None, + priority: Sequence[_spec.Source] | None = None, + **kwargs: Unpack[_spec.CryoCliArgs], +) -> pl.LazyFrame: + # get priority + if isinstance(priority, str): + priority = [priority] + if priority is None: + priority = ['memory', 'disk', 'rpc'] + + # attempt to load from priority targets + for p in priority: + if priority == 'memory': + if lfs is not None and datatype in lfs: + return lfs[datatype] + + elif priority == 'disk': + kwargs = _args.parse_cli_args(**kwargs) + if not cryo.is_data_saved(datatype=datatype, **kwargs): + if 'rpc' in priority: + cryo.freeze(datatype=datatype, **kwargs) + else: + raise Exception('data is missing from disk') + return _filesystem.scan(datatype=datatype, **kwargs) + + elif priority == 'rpc': + return cryo.collect( + datatype=datatype, output_format='polars', **kwargs + ).lazy() + + else: + raise Exception('invalid priority target: ' + str(priority)) + + raise Exception('could not load data from any priority targets: ' + str(priority)) diff --git a/crates/python/python/cryo/_spec.py b/crates/python/python/cryo/_spec.py index 11af8ae..7ab5ca7 100644 --- a/crates/python/python/cryo/_spec.py +++ b/crates/python/python/cryo/_spec.py @@ -2,17 +2,21 @@ import typing if typing.TYPE_CHECKING: from typing import Literal - from typing import TypedDict from typing import Union + from typing_extensions import TypedDict + + Source = Literal['memory', 'disk', 'rpc', 'db', 's3'] Datatype = str + BlockReference = Union[int | Literal['latest']] + FileFormat = Union[ Literal['parquet'], Literal['csv'], Literal['json'], - Literal['avro'], ] + PythonOutput = Union[ Literal['polars'], Literal['pandas'], @@ -54,4 +58,3 @@ if typing.TYPE_CHECKING: topic3: str | bytes | None inner_request_size: int | None no_verbose: bool - diff --git a/crates/python/python_tests/test_chunks.py b/crates/python/python_tests/test_chunks.py index 8303df3..8f81d13 100644 --- a/crates/python/python_tests/test_chunks.py +++ b/crates/python/python_tests/test_chunks.py @@ -1,4 +1,3 @@ - # def test_align(): # raise NotImplementedError() diff --git a/crates/python/python_tests/test_columns.py b/crates/python/python_tests/test_columns.py index a751b88..b276b29 100644 --- a/crates/python/python_tests/test_columns.py +++ b/crates/python/python_tests/test_columns.py @@ -1,4 +1,3 @@ - # def test_include_columns(): # raise NotImplementedError() diff --git a/crates/python/python_tests/test_datatypes.py b/crates/python/python_tests/test_datatypes.py index b3cc0d2..72652a7 100644 --- a/crates/python/python_tests/test_datatypes.py +++ b/crates/python/python_tests/test_datatypes.py @@ -48,4 +48,3 @@ def test_datatype_hex(datatype, blocks): # def test_sort(): # raise NotImplementedError() - diff --git a/crates/python/python_tests/test_output_formats.py b/crates/python/python_tests/test_output_formats.py index ccf6ae8..c9c3c22 100644 --- a/crates/python/python_tests/test_output_formats.py +++ b/crates/python/python_tests/test_output_formats.py @@ -55,4 +55,3 @@ def python_output_python_formats(query, format): output_format, output_type = format df = cryo.collect(output_format=output_format, **query) assert isinstance(df, output_type) -