diff --git a/.github/actions/rust-dependency-cooldown/.gitignore b/.github/actions/rust-dependency-cooldown/.gitignore deleted file mode 100644 index c18dd8d8..00000000 --- a/.github/actions/rust-dependency-cooldown/.gitignore +++ /dev/null @@ -1 +0,0 @@ -__pycache__/ diff --git a/.github/actions/rust-dependency-cooldown/README.md b/.github/actions/rust-dependency-cooldown/README.md deleted file mode 100644 index c36831f3..00000000 --- a/.github/actions/rust-dependency-cooldown/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Rust dependency cooldown - -This action checks newly locked crates.io versions against the caller's -`registry.global-min-publish-age` setting. S2-owned crates listed in -`first-party-crates.txt` are exempt from the publication-age check. - -## Security exceptions - -`security-exceptions.toml` records reviewed exceptions for individual security -releases. Each entry requires an exact crate name and version, an advisory -identifier or URL, and a reason: - -```toml -[[exception]] -crate = "rustls" -version = "0.23.45" -advisory = "RUSTSEC-2026-0285" -reason = "Fixes TLS 1.3 handshake messages accepted at the wrong encryption level" -``` - -Review the advisory and confirm that the exact release fixes it before adding an -entry. Ranges, wildcards, duplicate entries, and malformed configuration are -rejected. The action prints the crate, version, advisory, and reason whenever it -uses an exception. The exception waives only publication age; other dependency -checks, including `cargo deny`, continue to apply. - -The exception file is distributed with this action. After merging an exception, -update consuming repositories' pinned action or reusable-workflow commit to pick -it up. Future releases of the same crate still have to satisfy the cooldown. - -Entries need no expiry field: an approved release follows the normal age check -once it is old enough. Old entries may be removed in a later cleanup. An empty -file or `exception = []` means there are no security exceptions. - -Cargo also enforces publication age during dependency resolution. To select an -approved release, override that resolver check for the targeted update command: - -```sh -CARGO_RESOLVER_INCOMPATIBLE_PUBLISH_AGE=allow cargo +nightly update -p rustls --precise 0.23.45 -``` - -Run the cooldown action on the resulting lockfile to check every newly selected -version against the exception list and normal age requirement. - -## Tests - -Run from the repository root: - -```sh -python3 -m unittest discover -s .github/actions/rust-dependency-cooldown -p 'test_*.py' -v -``` diff --git a/.github/actions/rust-dependency-cooldown/check.py b/.github/actions/rust-dependency-cooldown/check.py index 53f7d0b0..535e2884 100644 --- a/.github/actions/rust-dependency-cooldown/check.py +++ b/.github/actions/rust-dependency-cooldown/check.py @@ -20,7 +20,6 @@ ACTION_ROOT = Path(__file__).resolve().parent REPO_ROOT = Path.cwd() CONFIG = REPO_ROOT / ".cargo" / "config.toml" ALLOWLIST = ACTION_ROOT / "first-party-crates.txt" -SECURITY_EXCEPTIONS = ACTION_ROOT / "security-exceptions.toml" CRATES_IO_SOURCE = "registry+https://github.com/rust-lang/crates.io-index" INDEX_BASE = "https://index.crates.io" USER_AGENT = "s2 minimum-publish-age check (github.com/s2-streamstore/s2)" @@ -70,36 +69,6 @@ def allowed_crates() -> set[str]: } -def security_exceptions() -> dict[tuple[str, str], dict[str, str]]: - with SECURITY_EXCEPTIONS.open("rb") as exceptions_file: - document = tomllib.load(exceptions_file) - if document.keys() - {"exception"}: - raise ValueError(f"unexpected fields in {SECURITY_EXCEPTIONS}") - entries = document.get("exception", []) - if not isinstance(entries, list): - raise ValueError(f"expected [[exception]] entries in {SECURITY_EXCEPTIONS}") - required_fields = {"crate", "version", "advisory", "reason"} - exceptions: dict[tuple[str, str], dict[str, str]] = {} - for index, entry in enumerate(entries, start=1): - if not isinstance(entry, dict) or entry.keys() != required_fields: - raise ValueError(f"security exception {index} must contain {sorted(required_fields)}") - for field, value in entry.items(): - if not isinstance(value, str) or not value or value != value.strip(): - raise ValueError(f"security exception {index} has invalid {field}") - if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]*", entry["crate"]) is None: - raise ValueError(f"security exception {index} requires an exact crate name") - if re.fullmatch( - r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?", - entry["version"], - ) is None: - raise ValueError(f"security exception {index} requires an exact version") - key = (entry["crate"], entry["version"]) - if key in exceptions: - raise ValueError(f"duplicate security exception for {key[0]} {key[1]}") - exceptions[key] = entry - return exceptions - - def crates_io_versions(lock_text: str) -> set[tuple[str, str]]: packages = tomllib.loads(lock_text).get("package", []) return { @@ -227,7 +196,6 @@ def main() -> int: try: if args.base_ref and run_git("rev-parse", "--verify", "--quiet", args.base_ref).returncode: raise ValueError(f"base ref {args.base_ref!r} is not available") - approved_exceptions = security_exceptions() if args.base_ref and not has_relevant_changes(args.base_ref): print("No Rust dependency cooldown files changed; check skipped.") return 0 @@ -263,12 +231,6 @@ def main() -> int: checked += 1 crate_age = now - pubtime if crate_age < age_limit: - if exception := approved_exceptions.get((name, version)): - print( - f"Publication cooldown waived for {name} {version} ({lockfile}): " - f"{exception['advisory']} — {exception['reason']}" - ) - continue violations.append( f"{name} {version} ({lockfile}): published " f"{crate_age.total_seconds() / 86400:.1f} days ago; " @@ -284,7 +246,7 @@ def main() -> int: print(f" - {violation}", file=sys.stderr) return 1 - print(f"Checked {checked} new crates.io version(s); publication cooldown policy satisfied.") + print(f"Checked {checked} new crates.io version(s); all are at least {age_text} old.") return 0 diff --git a/.github/actions/rust-dependency-cooldown/security-exceptions.toml b/.github/actions/rust-dependency-cooldown/security-exceptions.toml deleted file mode 100644 index c0e3a044..00000000 --- a/.github/actions/rust-dependency-cooldown/security-exceptions.toml +++ /dev/null @@ -1,5 +0,0 @@ -[[exception]] -crate = "rustls" -version = "0.23.45" -advisory = "RUSTSEC-2026-0285" -reason = "Fixes TLS 1.3 handshake messages accepted at the wrong encryption level" diff --git a/.github/actions/rust-dependency-cooldown/test_check.py b/.github/actions/rust-dependency-cooldown/test_check.py deleted file mode 100644 index 313e59a5..00000000 --- a/.github/actions/rust-dependency-cooldown/test_check.py +++ /dev/null @@ -1,290 +0,0 @@ -from __future__ import annotations - -import importlib.util -import io -import json -import subprocess -import sys -import tempfile -import unittest -from contextlib import ExitStack, redirect_stderr, redirect_stdout -from datetime import datetime, timedelta, timezone -from pathlib import Path -from unittest.mock import patch - - -SPEC = importlib.util.spec_from_file_location( - "cooldown_check", Path(__file__).with_name("check.py") -) -assert SPEC is not None and SPEC.loader is not None -check = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(check) - -NOW = datetime(2026, 9, 16, tzinfo=timezone.utc) -APPROVED_EXCEPTION = { - "crate": "rustls", - "version": "0.23.45", - "advisory": "https://rustsec.org/advisories/RUSTSEC-2026-0285.html", - "reason": "First release fixing TLS 1.3 handshake encryption-level validation.", -} - - -def exception_document(*entries: dict[str, object]) -> str: - return "\n".join( - "[[exception]]\n" - + "\n".join(f"{name} = {json.dumps(value)}" for name, value in entry.items()) - + "\n" - for entry in entries - ) - - -class CooldownSecurityExceptionTests(unittest.TestCase): - def setUp(self) -> None: - temporary_directory = tempfile.TemporaryDirectory() - self.addCleanup(temporary_directory.cleanup) - temporary_root = Path(temporary_directory.name) - self.repo = temporary_root / "consumer" - self.repo.mkdir() - self.action = temporary_root / "trusted-action" - self.action.mkdir() - self.exceptions = self.action / "security-exceptions.toml" - self.exceptions.write_text(exception_document(APPROVED_EXCEPTION)) - (self.action / "first-party-crates.txt").write_text("s2-api\n") - (self.repo / ".cargo").mkdir() - (self.repo / ".cargo" / "config.toml").write_text( - '[registry]\nglobal-min-publish-age = "7 days"\n' - ) - self.write_lock() - self.git("init", "--quiet") - self.commit() - - def git(self, *arguments: str) -> None: - subprocess.run( - [ - "git", - "-c", - "user.name=Cooldown Tests", - "-c", - "user.email=cooldown-tests@example.invalid", - "-c", - "commit.gpgsign=false", - "-c", - "core.hooksPath=/dev/null", - *arguments, - ], - cwd=self.repo, - check=True, - capture_output=True, - text=True, - ) - - def commit(self) -> None: - self.git("add", ".") - self.git("commit", "--quiet", "-m", "test: record baseline") - - def write_lock(self, *packages: tuple[str, str, str]) -> None: - contents = "version = 3\n" - for name, version, source in packages: - contents += ( - "\n[[package]]\n" - f"name = {json.dumps(name)}\n" - f"version = {json.dumps(version)}\n" - f"source = {json.dumps(source)}\n" - ) - (self.repo / "Cargo.lock").write_text(contents) - - def run_gate( - self, - publication_times: dict[str, dict[str, datetime]] | None = None, - ) -> tuple[int, str, str, list[str]]: - published = publication_times or { - "rustls": {"0.23.45": NOW - timedelta(days=1)}, - } - output, errors = io.StringIO(), io.StringIO() - with ExitStack() as stack: - for name, value in { - "ACTION_ROOT": self.action, - "REPO_ROOT": self.repo, - "CONFIG": self.repo / ".cargo" / "config.toml", - "ALLOWLIST": self.action / "first-party-crates.txt", - "SECURITY_EXCEPTIONS": self.exceptions, - }.items(): - stack.enter_context(patch.object(check, name, value)) - stack.enter_context( - patch.object(sys, "argv", ["check.py", "--repo-root", str(self.repo), "HEAD"]) - ) - clock = stack.enter_context(patch.object(check, "datetime", wraps=datetime)) - clock.now.return_value = NOW - lookup = stack.enter_context( - patch.object(check, "publication_times", side_effect=published.__getitem__) - ) - stack.enter_context(redirect_stdout(output)) - stack.enter_context(redirect_stderr(errors)) - status = check.main() - lookups = [call.args[0] for call in lookup.call_args_list] - return status, output.getvalue(), errors.getvalue(), lookups - - def test_exact_security_exception_passes_and_reports_justification(self) -> None: - self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) - status, output, errors, _ = self.run_gate() - self.assertEqual(status, 0, errors) - for field in APPROVED_EXCEPTION.values(): - self.assertIn(field, output) - - def test_exception_does_not_cover_another_version_or_crate(self) -> None: - for name, version in [("rustls", "0.23.46"), ("another-crate", "0.23.45")]: - with self.subTest(crate=name, version=version): - self.write_lock((name, version, check.CRATES_IO_SOURCE)) - status, _, errors, _ = self.run_gate( - {name: {version: NOW - timedelta(days=1)}} - ) - self.assertEqual(status, 1) - self.assertIn(f"{name} {version}", errors) - - def test_exception_does_not_hide_an_unrelated_young_dependency(self) -> None: - self.write_lock( - ("rustls", "0.23.45", check.CRATES_IO_SOURCE), - ("another-crate", "1.0.0", check.CRATES_IO_SOURCE), - ) - status, output, errors, _ = self.run_gate( - { - "rustls": {"0.23.45": NOW - timedelta(days=1)}, - "another-crate": {"1.0.0": NOW - timedelta(days=1)}, - } - ) - self.assertEqual(status, 1) - self.assertIn(APPROVED_EXCEPTION["advisory"], output) - self.assertIn("another-crate 1.0.0", errors) - self.assertNotIn("rustls 0.23.45", errors) - - def test_consumer_repository_cannot_supply_its_own_exception(self) -> None: - unapproved = {**APPROVED_EXCEPTION, "version": "0.23.46"} - consumer_action = self.repo / ".github" / "actions" / "rust-dependency-cooldown" - consumer_action.mkdir(parents=True) - for directory in [self.repo, self.repo / ".cargo", consumer_action]: - (directory / "security-exceptions.toml").write_text(exception_document(unapproved)) - self.write_lock(("rustls", "0.23.46", check.CRATES_IO_SOURCE)) - status, _, errors, _ = self.run_gate( - {"rustls": {"0.23.46": NOW - timedelta(days=1)}} - ) - self.assertEqual(status, 1) - self.assertIn("rustls 0.23.46", errors) - - def test_malformed_security_exceptions_fail_closed(self) -> None: - documents = { - "invalid TOML": "[[exception]", - "not an array": "exception = 1\n", - "not a table": "exception = [1]\n", - "unknown top-level field": "exceptions = []\n", - } - for field in APPROVED_EXCEPTION: - missing = {name: value for name, value in APPROVED_EXCEPTION.items() if name != field} - documents[f"missing {field}"] = exception_document(missing) - for value in ["", " ", 123, f" {APPROVED_EXCEPTION[field]}"]: - documents[f"invalid {field}: {value!r}"] = exception_document( - {**APPROVED_EXCEPTION, field: value} - ) - documents["unknown field"] = exception_document({**APPROVED_EXCEPTION, "extra": "value"}) - documents["duplicate pair"] = exception_document(APPROVED_EXCEPTION, APPROVED_EXCEPTION) - self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) - for description, document in documents.items(): - with self.subTest(description=description): - self.exceptions.write_text(document) - status, _, errors, _ = self.run_gate() - self.assertEqual(status, 2) - self.assertIn("minimum-publish-age error", errors) - - def test_missing_action_exception_file_fails_closed(self) -> None: - self.exceptions.unlink() - self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) - status, _, errors, _ = self.run_gate() - self.assertEqual(status, 2) - self.assertIn("minimum-publish-age error", errors) - - def test_malformed_exceptions_fail_even_when_no_lockfile_changed(self) -> None: - self.exceptions.write_text("[[exception]") - status, _, errors, _ = self.run_gate() - self.assertEqual(status, 2) - self.assertIn("minimum-publish-age error", errors) - - def test_empty_exception_lists_do_not_exempt_young_dependencies(self) -> None: - self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) - for document in ["", "exception = []\n"]: - with self.subTest(document=document): - self.exceptions.write_text(document) - status, _, errors, _ = self.run_gate() - self.assertEqual(status, 1) - self.assertIn("rustls 0.23.45", errors) - - def test_exception_ranges_and_wildcards_are_rejected(self) -> None: - self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) - for field, value in [ - ("crate", "rustls*"), - ("version", "*"), - ("version", "0.23.*"), - ("version", ">=0.23.45"), - ]: - with self.subTest(field=field, value=value): - self.exceptions.write_text( - exception_document({**APPROVED_EXCEPTION, field: value}) - ) - status, _, errors, _ = self.run_gate() - self.assertEqual(status, 2) - self.assertIn("minimum-publish-age error", errors) - - def test_approved_version_still_requires_a_publication_time(self) -> None: - self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) - status, _, errors, _ = self.run_gate({"rustls": {}}) - self.assertEqual(status, 2) - self.assertIn("no publication time for rustls 0.23.45", errors) - - def test_mature_approved_version_uses_normal_age_check(self) -> None: - self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) - status, output, errors, lookups = self.run_gate( - {"rustls": {"0.23.45": NOW - timedelta(days=7)}} - ) - self.assertEqual(status, 0, errors) - self.assertNotIn(APPROVED_EXCEPTION["advisory"], output) - self.assertEqual(lookups, ["rustls"]) - - def test_age_boundary_for_ordinary_dependencies_is_unchanged(self) -> None: - self.write_lock(("another-crate", "1.0.0", check.CRATES_IO_SOURCE)) - for age, expected_status in [(timedelta(days=7), 0), (timedelta(days=7, seconds=-1), 1)]: - with self.subTest(age=age): - status, _, errors, _ = self.run_gate({"another-crate": {"1.0.0": NOW - age}}) - self.assertEqual(status, expected_status, errors) - - def test_first_party_allowlist_still_exempts_new_versions(self) -> None: - self.write_lock(("s2-api", "99.0.0", check.CRATES_IO_SOURCE)) - status, _, errors, lookups = self.run_gate() - self.assertEqual(status, 0, errors) - self.assertEqual(lookups, []) - - def test_other_sources_are_not_processed_as_security_exceptions(self) -> None: - for source in [ - "registry+https://example.invalid/index", - "git+https://example.invalid/rustls", - ]: - with self.subTest(source=source): - self.write_lock(("rustls", "0.23.45", source)) - status, output, errors, lookups = self.run_gate() - self.assertEqual(status, 0, errors) - self.assertNotIn(APPROVED_EXCEPTION["advisory"], output) - self.assertEqual(lookups, []) - - def test_existing_locked_versions_are_not_rechecked(self) -> None: - self.write_lock(("already-locked", "1.0.0", check.CRATES_IO_SOURCE)) - self.commit() - self.write_lock( - ("already-locked", "1.0.0", check.CRATES_IO_SOURCE), - ("another-crate", "1.0.0", check.CRATES_IO_SOURCE), - ) - status, _, errors, lookups = self.run_gate( - {"another-crate": {"1.0.0": NOW - timedelta(days=8)}} - ) - self.assertEqual(status, 0, errors) - self.assertEqual(lookups, ["another-crate"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2da7b71d..7c8a043e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -310,18 +310,6 @@ jobs: exit 1 fi - - name: Test with persistent volume and WAL storage - run: | - helm template test-release charts/s2-lite-helm \ - --set persistentVolume.enabled=true \ - --set walStorage.persistentVolume.enabled=true \ - --dry-run > /dev/null - helm template test-release charts/s2-lite-helm \ - --set objectStorage.enabled=true \ - --set objectStorage.bucket=test-bucket \ - --set walStorage.bucket=test-wal-bucket \ - --dry-run > /dev/null - build-server: name: Build s2-lite needs: rust-dependency-cooldown diff --git a/.github/workflows/rust-dependency-cooldown-tests.yml b/.github/workflows/rust-dependency-cooldown-tests.yml deleted file mode 100644 index 76f56ecc..00000000 --- a/.github/workflows/rust-dependency-cooldown-tests.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Rust dependency cooldown tests - -on: - pull_request: - paths: - - .github/actions/rust-dependency-cooldown/** - - .github/workflows/rust-dependency-cooldown-tests.yml - push: - branches: [main] - paths: - - .github/actions/rust-dependency-cooldown/** - - .github/workflows/rust-dependency-cooldown-tests.yml - -permissions: - contents: read - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - run: python3 -m unittest discover -s .github/actions/rust-dependency-cooldown -p 'test_*.py' -v diff --git a/Cargo.lock b/Cargo.lock index de3da95d..9fa1927f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,7 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ "crypto-common 0.2.2", - "inout", + "inout 0.2.2", ] [[package]] @@ -39,36 +39,46 @@ dependencies = [ [[package]] name = "aes" -version = "0.9.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cipher", + "cfg-if", + "cipher 0.4.4", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" +dependencies = [ + "cipher 0.5.2", "cpubits", - "cpufeatures 0.3.1", - "zeroize", + "cpufeatures 0.3.0", ] [[package]] name = "aes-gcm" -version = "0.11.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f" +checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" dependencies = [ "aead", - "aes", - "cipher", + "aes 0.9.2", + "cipher 0.5.2", "ctr", - "ctutils", "ghash", + "subtle", "zeroize", ] [[package]] name = "aho-corasick" -version = "1.1.5" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -87,9 +97,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" dependencies = [ "libc", ] @@ -152,9 +162,9 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "apple-native-keyring-store" -version = "1.0.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b350bfd03649e07aa05c0a81b3e15934374e585c98204a57e20b9d49f49bb9a" +checksum = "797f94b6a53d7d10b56dc18290e0d40a2158352f108bb4ff32350825081a9f29" dependencies = [ "keyring-core", "log", @@ -176,6 +186,12 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + [[package]] name = "arrayvec" version = "0.7.8" @@ -194,7 +210,7 @@ dependencies = [ "nom", "num-traits", "rusticata-macros", - "thiserror 2.0.20", + "thiserror 2.0.19", "time", ] @@ -284,9 +300,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.46" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f10dafd0c8d2e51ae9a748805777613ed0bbe17bf586b76c8311f45c020a32f" +checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8" dependencies = [ "compression-codecs", "compression-core", @@ -414,21 +430,15 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.92" +version = "0.1.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] -[[package]] -name = "asyncband" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2d85fd3d291fabcc40c7232c92c280ec1754fd7b5d7ea769f143222143e179a" - [[package]] name = "atomic" version = "0.6.1" @@ -452,9 +462,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.12.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" +checksum = "1b180a3c8b55960db3426d8964b8745e652466a1a49fe1a2eda828046d30b5e4" dependencies = [ "aws-credential-types", "aws-runtime", @@ -495,9 +505,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.18.1" +version = "1.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" +checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -506,9 +516,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.45.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" +checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" dependencies = [ "cc", "cmake", @@ -519,9 +529,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.9.2" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef47857a1d4488b528f4a5d5715fa7c3300820897824152234d3fa22b1426657" +checksum = "c9007227e10b5fed2f3e0a2beff489211e2b5604c400b7a9d5d81ca9d64c24bb" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -544,9 +554,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.109.0" +version = "1.105.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3cfe74df5d9ad2fedd691973ad3521ebf4f27a3c68c792556686aedb5519bab" +checksum = "6ffd0fbe7873cb548a7aa60f9573c268fff94155397fd4f14dc9f1ecaaab8516" dependencies = [ "arc-swap", "aws-credential-types", @@ -570,9 +580,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.111.0" +version = "1.107.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b0ec31ed6191bd11350aae4b2004198f2db21350cb0a20c57e0a92e55dd161" +checksum = "175763eb222a46377df7aa257a3bca980ab3e96703fefc8f4d0b8da6ad2e254c" dependencies = [ "arc-swap", "aws-credential-types", @@ -596,9 +606,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.114.0" +version = "1.110.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef45745026107ec30c4ef86bd8ae4b002e7e5f6a86e4225240bdf6b06a0b944a" +checksum = "dd8b14781dfbff48984017d57167b6ea0b6471c6920ec52b44a2677c7feb3c13" dependencies = [ "arc-swap", "aws-credential-types", @@ -634,11 +644,11 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", "http 1.5.0", "percent-encoding", - "sha2", + "sha2 0.11.0", "time", "tracing", ] @@ -677,9 +687,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.4.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebfd138fac0337cee7516c352757ea73b9f2266e57d0bcb5bc70e9547e45aef1" +checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -734,9 +744,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.14.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b82e438d30e02a825d363bd639a9efaed68a8089d86101054b0081e7e0d3e606" +checksum = "07505b34e8f4b3591a4fa69e9792b52289b95488dbbc68c3c0075b7bedb245e1" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -760,9 +770,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.16.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223" +checksum = "3b98f2e1fd67ec06618f9c291e5e495a468e60519e44c9c1979cd0521f3affdb" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -800,9 +810,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.6.3" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9" +checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8" dependencies = [ "base64-simd", "bytes", @@ -835,9 +845,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.6.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" +checksum = "eec1cd5469f328c782dc3e33d4153cf118a54e33cbb3356d60d16f89883e1f94" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -954,7 +964,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide 0.8.9", + "miniz_oxide", "object", "rustc-demangle", "windows-link 0.2.1", @@ -975,12 +985,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" - [[package]] name = "base64-simd" version = "0.8.0" @@ -1032,30 +1036,25 @@ dependencies = [ [[package]] name = "bitflags" -version = "1.3.2" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" dependencies = [ "serde_core", ] [[package]] name = "blake3" -version = "1.8.7" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" dependencies = [ + "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", ] [[package]] @@ -1078,18 +1077,18 @@ dependencies = [ [[package]] name = "block-padding" -version = "0.4.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "hybrid-array", + "generic-array", ] [[package]] name = "blocking" -version = "1.7.0" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" dependencies = [ "async-channel", "async-task", @@ -1100,13 +1099,13 @@ dependencies = [ [[package]] name = "bollard" -version = "0.21.1" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbe8358268799ebb3e4df23cb9d47f4c72bbc4f5247e2fa6a1bf7b6c0baea220" +checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4" dependencies = [ "async-stream", - "base64 0.22.1", - "bitflags 2.13.2", + "base64", + "bitflags", "bollard-buildkit-proto", "bollard-stubs", "bytes", @@ -1124,7 +1123,7 @@ dependencies = [ "log", "num", "pin-project-lite", - "rand 0.10.2", + "rand 0.9.5", "rustls", "rustls-native-certs", "rustls-pki-types", @@ -1132,7 +1131,7 @@ dependencies = [ "serde_derive", "serde_json", "serde_urlencoded", - "thiserror 2.0.20", + "thiserror 2.0.19", "time", "tokio", "tokio-stream", @@ -1145,23 +1144,24 @@ dependencies = [ [[package]] name = "bollard-buildkit-proto" -version = "0.8.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5c97450e79c7c565302dd92e86b08823b47550fcb4fc5ce910194d1b087a1a3" +checksum = "85a885520bf6249ab931a764ffdb87b0ceef48e6e7d807cfdb21b751e086e1ad" dependencies = [ "prost", "prost-types", "tonic", "tonic-prost", + "ureq", ] [[package]] name = "bollard-stubs" -version = "1.53.1-rc.29.3.1" +version = "1.52.1-rc.29.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce412eb6f7096743011dc3cb5c674caeb24ced61d8c498fe07cf7998a4fea889" +checksum = "0f0a8ca8799131c1837d1282c3f81f31e76ceb0ce426e04a7fe1ccee3287c066" dependencies = [ - "base64 0.22.1", + "base64", "bollard-buildkit-proto", "bytes", "prost", @@ -1182,9 +1182,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.13.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530" dependencies = [ "memchr", "regex-automata", @@ -1199,9 +1199,9 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytecheck" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26333eeac754f0ad8a6bcd0eb0ac012156302e4e16b852b72ee399aea4f12c29" +checksum = "0caa33a2c0edca0419d15ac723dff03f1956f7978329b1e3b5fdaaaed9d3ca8b" dependencies = [ "bytecheck_derive", "ptr_meta", @@ -1211,13 +1211,13 @@ dependencies = [ [[package]] name = "bytecheck_derive" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46d07918caa9eeaaf06b7873925c53a61daac173539b4f7715090745e44e4e69" +checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", ] [[package]] @@ -1259,9 +1259,9 @@ dependencies = [ [[package]] name = "bytesize" -version = "2.7.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7354288c522e7e980fafd2075d63d1285794c3a6a16cdd492f189ea406e5f18b" +checksum = "351a3e803ee3c6eaeee6b00076b767514b37c32a73d326c3ec7abddb7d6c3493" [[package]] name = "castaway" @@ -1274,18 +1274,18 @@ dependencies = [ [[package]] name = "cbc" -version = "0.2.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" dependencies = [ - "cipher", + "cipher 0.4.4", ] [[package]] name = "cc" -version = "1.4.5" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -1307,12 +1307,12 @@ checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" -version = "0.10.2" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", "rand_core 0.10.1", ] @@ -1330,6 +1330,16 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout 0.1.4", +] + [[package]] name = "cipher" version = "0.5.2" @@ -1338,14 +1348,14 @@ checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ "block-buffer 0.12.1", "crypto-common 0.2.2", - "inout", + "inout 0.2.2", ] [[package]] name = "clap" -version = "4.6.6" +version = "4.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" dependencies = [ "clap_builder", "clap_derive", @@ -1353,9 +1363,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.6" +version = "4.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" dependencies = [ "anstream", "anstyle", @@ -1372,7 +1382,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -1396,6 +1406,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "cmsketch" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7ee2cfacbd29706479902b06d75ad8f1362900836aa32799eabc7e004bfd854" + [[package]] name = "color-print" version = "0.3.7" @@ -1429,14 +1445,14 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] name = "combine" -version = "4.6.8" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ "bytes", "memchr", @@ -1459,22 +1475,22 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.41" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58a6d0db8759036a783bc7c3f7a07f8cef3bf9470eb1db3bc86e8bcd1c5d0fe8" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" dependencies = [ "compression-core", "flate2", "memchr", - "zstd 0.14.0", - "zstd-safe 8.0.0", + "zstd", + "zstd-safe", ] [[package]] name = "compression-core" -version = "0.4.33" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" [[package]] name = "concurrent-queue" @@ -1500,16 +1516,16 @@ dependencies = [ "serde-untagged", "serde_core", "serde_json", - "toml 1.1.6+spec-1.1.0", + "toml 1.1.4+spec-1.1.0", "winnow 1.0.4", "yaml-rust2", ] [[package]] name = "console" -version = "0.16.6" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" dependencies = [ "encode_unicode", "libc", @@ -1585,12 +1601,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "core_detect" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" - [[package]] name = "cpubits" version = "0.1.1" @@ -1608,9 +1618,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] @@ -1627,18 +1637,18 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-epoch" -version = "0.9.21" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] @@ -1655,9 +1665,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.23" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crunchy" @@ -1692,7 +1702,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" dependencies = [ - "cipher", + "cipher 0.5.2", ] [[package]] @@ -1716,12 +1726,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.24.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core 0.24.1", - "darling_macro 0.24.1", + "darling_core 0.23.0", + "darling_macro 0.23.0", ] [[package]] @@ -1739,15 +1749,15 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.24.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ "ident_case", "proc-macro2", "quote", "strsim", - "syn 3.0.5", + "syn 2.0.119", ] [[package]] @@ -1763,13 +1773,13 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.24.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core 0.24.1", + "darling_core 0.23.0", "quote", - "syn 3.0.5", + "syn 2.0.119", ] [[package]] @@ -1788,46 +1798,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.11.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" - -[[package]] -name = "datasketches" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" - -[[package]] -name = "defmt" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" -dependencies = [ - "bitflags 1.3.2", - "defmt-macros", -] - -[[package]] -name = "defmt-macros" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" -dependencies = [ - "defmt-parser", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "defmt-parser" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" -dependencies = [ - "thiserror 2.0.20", -] +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "der-parser" @@ -1866,6 +1839,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "crypto-common 0.1.7", + "subtle", ] [[package]] @@ -1882,9 +1856,9 @@ dependencies = [ [[package]] name = "dirs" -version = "7.0.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d57d423b3c82e89b9a24ca3091fee61f456a26edbd28d26c65906f4bc1dcd8f" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ "dirs-sys", ] @@ -1898,7 +1872,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1909,7 +1883,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -1927,7 +1901,7 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d" dependencies = [ - "base64 0.22.1", + "base64", "serde", "serde_json", ] @@ -1952,7 +1926,7 @@ checksum = "f88959de2d447fd3eddcf1909d1f19fe084e27a056a6904203dc5d8b9e771c1e" dependencies = [ "rust_decimal", "serde", - "thiserror 2.0.20", + "thiserror 2.0.19", "time", "winnow 0.6.26", ] @@ -1965,9 +1939,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" -version = "1.18.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" [[package]] name = "encode_unicode" @@ -1977,17 +1951,11 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "encoding_rs" -version = "0.8.41" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5ef0006ac9ab233c38522f5ae99cae3625151de8f706cacee1cba4b8e2832a" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", - "core_detect", - "multiversion", - "multiversion_no_op", - "rustversion", - "scopeguard", - "simdutf8", ] [[package]] @@ -2062,7 +2030,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2097,11 +2065,10 @@ dependencies = [ [[package]] name = "eyre" -version = "0.6.14" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08309dbcc659c5549a24ddb9b27027640641b282ef5768267c7e675558986a3" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" dependencies = [ - "autocfg", "indenter", "once_cell", ] @@ -2163,9 +2130,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.12" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "fixedbitset" @@ -2179,18 +2146,18 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.13.2", + "bitflags", "rustc_version", ] [[package]] name = "flate2" -version = "1.1.10" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", - "miniz_oxide 0.9.1", + "miniz_oxide", "zlib-rs", ] @@ -2232,18 +2199,18 @@ dependencies = [ [[package]] name = "foyer" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a59f42276891c0a4ce683fcda1aa1b4fd1065d68116c264e4bf49b9d318a0ed" +checksum = "3b0abc0b87814989efa711f9becd9f26969820e2d3905db27d10969c4bd45890" dependencies = [ "anyhow", - "asyncband", "equivalent", "foyer-common", "foyer-memory", "foyer-storage", "foyer-tokio", "futures-util", + "mea", "mixtrics", "pin-project", "serde", @@ -2252,9 +2219,9 @@ dependencies = [ [[package]] name = "foyer-common" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643b47d510032a01e5af70dca288b0c5ec531646c1a8392e793fb5b0c13bef30" +checksum = "a3db80d5dece93adb7ad709c84578794724a9cba342a7e566c3551c7ec626789" dependencies = [ "anyhow", "bincode", @@ -2279,21 +2246,21 @@ dependencies = [ [[package]] name = "foyer-memory" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae51f5089f3d9025ae77f17ea66d2489ac0f82fbb1d91462807bea174d486d82" +checksum = "db907f40a527ca2aa2f40a5f68b32ea58aa70f050cd233518e9ffd402cfba6ce" dependencies = [ "anyhow", - "asyncband", - "bitflags 2.13.2", - "datasketches", + "bitflags", + "cmsketch", "equivalent", "foyer-common", "foyer-intrusive-collections", "foyer-tokio", "futures-util", - "hashbrown 0.17.1", - "itertools 0.15.0", + "hashbrown 0.16.1", + "itertools 0.14.0", + "mea", "mixtrics", "parking_lot", "paste", @@ -2304,13 +2271,12 @@ dependencies = [ [[package]] name = "foyer-storage" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "118192f532ba013f0cdc607efc22324b9ed855baed185608af0daa986e835c6b" +checksum = "1983f1db3d0710e9c9d5fc116d9202dccd41a2d1e032572224f1aff5520aa958" dependencies = [ "allocator-api2", "anyhow", - "asyncband", "bytes", "core_affinity", "equivalent", @@ -2321,25 +2287,26 @@ dependencies = [ "fs4", "futures-core", "futures-util", - "hashbrown 0.17.1", + "hashbrown 0.16.1", "io-uring", - "itertools 0.15.0", + "itertools 0.14.0", "libc", "lz4", + "mea", "parking_lot", "pin-project", - "rand 0.10.2", + "rand 0.9.5", "serde", "tracing", "twox-hash", - "zstd 0.13.3", + "zstd", ] [[package]] name = "foyer-tokio" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6741d1133dfaab64d3f8a9290bbfed3685084add28f8b6ee7a6264aa4dc26918" +checksum = "f6577b05a7ffad0db555aedf00bfe52af818220fc4c1c3a7a12520896fc38627" dependencies = [ "tokio", ] @@ -2382,9 +2349,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" dependencies = [ "futures-channel", "futures-core", @@ -2397,9 +2364,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", "futures-sink", @@ -2407,15 +2374,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-executor" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" dependencies = [ "futures-core", "futures-task", @@ -2424,9 +2391,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" [[package]] name = "futures-lite" @@ -2443,26 +2410,26 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", ] [[package]] name = "futures-sink" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-timer" @@ -2472,9 +2439,9 @@ checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-channel", "futures-core", @@ -2543,7 +2510,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" dependencies = [ "polyval", - "zeroize", ] [[package]] @@ -2572,9 +2538,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.19" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", @@ -2582,7 +2548,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.5.0", - "indexmap 2.14.2", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", @@ -2616,6 +2582,8 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.2.0", ] @@ -2653,9 +2621,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.5.3" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "hex" @@ -2665,11 +2633,20 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hkdf" -version = "0.13.0" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", ] [[package]] @@ -2734,9 +2711,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.5" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", @@ -2765,18 +2742,18 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" -version = "0.4.15" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] [[package]] name = "hyper" -version = "1.11.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ "atomic-waker", "bytes", @@ -2843,7 +2820,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", "futures-util", @@ -2901,9 +2878,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", @@ -2915,43 +2892,22 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", - "serde", "tinystr", "writeable", "zerovec", ] -[[package]] -name = "icu_locale_fallback" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" -dependencies = [ - "icu_locale_core", - "icu_locale_fallback_data", - "icu_provider", - "potential_utf", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locale_fallback_data" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" - [[package]] name = "icu_normalizer" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -2963,17 +2919,16 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ - "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -2984,20 +2939,18 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.3.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", - "serde", - "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -3005,28 +2958,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "icu_segmenter" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82d07aafccd67af15d02512a6adf5896fbc5ed00f2e99b471d2efa14016db3db" -dependencies = [ - "icu_collections", - "icu_locale_fallback", - "icu_provider", - "icu_segmenter_data", - "potential_utf", - "smallvec", - "utf8_iter", - "zerovec", -] - -[[package]] -name = "icu_segmenter_data" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae293c039020f9ec10710af98d29ce6aa2051486638b49c9a6409f3b4a9e98ad" - [[package]] name = "ident_case" version = "1.0.1" @@ -3073,9 +3004,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.2" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -3102,32 +3033,41 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "inout" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" dependencies = [ - "block-padding", "hybrid-array", ] [[package]] name = "io-uring" -version = "0.7.15" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" +checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0" dependencies = [ - "bitflags 2.13.2", + "bitflags", "cfg-if", "libc", ] [[package]] name = "ipnet" -version = "2.12.2" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "is_ci" @@ -3165,59 +3105,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "jiff" -version = "0.2.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" -dependencies = [ - "defmt", - "jiff-core", - "jiff-static", - "jiff-tzdb-platform", - "log", - "portable-atomic", - "portable-atomic-util", - "serde_core", - "windows-link 0.2.1", -] - -[[package]] -name = "jiff-core" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" -dependencies = [ - "defmt", -] - -[[package]] -name = "jiff-static" -version = "0.2.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" -dependencies = [ - "jiff-core", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "jiff-tzdb" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" - -[[package]] -name = "jiff-tzdb-platform" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" -dependencies = [ - "jiff-tzdb", -] - [[package]] name = "jni" version = "0.22.4" @@ -3230,7 +3117,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.20", + "thiserror 2.0.19", "walkdir", "windows-link 0.2.1", ] @@ -3279,9 +3166,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.105" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ "cfg-if", "futures-util", @@ -3301,9 +3188,9 @@ dependencies = [ [[package]] name = "json_to_table" -version = "0.14.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec64c9908ffa97b6ef6044d96c9f56de1c81643d3b9fbc2823e8847ab11b467" +checksum = "1f91246cf42b255a705436e360ca5894e376a50fdf696e4e60cb1bb6dd648e21" dependencies = [ "serde_json", "tabled", @@ -3311,9 +3198,9 @@ dependencies = [ [[package]] name = "keyring" -version = "4.2.0" +version = "4.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2270074a3d26bcac93c1dc5d2845eb4c089e8d761ccf6e0ea266a16004640627" +checksum = "72585bb6cc9bc370d1d545b7e23fcce71dfd4461c5e15275e3cf51bdfd9a980a" dependencies = [ "apple-native-keyring-store", "keyring-core", @@ -3344,9 +3231,9 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libredox" -version = "0.1.23" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed" +checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" dependencies = [ "libc", ] @@ -3365,9 +3252,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" @@ -3380,15 +3267,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.34" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lru" -version = "0.18.4" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" +checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" dependencies = [ "hashbrown 0.17.1", ] @@ -3452,6 +3339,15 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "mea" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31fc7d159de0085ab6dd7ff145a9819442cfd3d098f783263120503c3f3e58b0" +dependencies = [ + "slab", +] + [[package]] name = "memchr" version = "2.8.3" @@ -3514,15 +3410,6 @@ name = "miniz_oxide" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", -] - -[[package]] -name = "miniz_oxide" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -3530,9 +3417,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.3" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi", @@ -3555,33 +3442,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" -[[package]] -name = "multiversion" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ca4bea16ffc3f443cf7d866912118196bfef4c6a1556ca00f9f9b00bb43f7c" -dependencies = [ - "multiversion-macros", -] - -[[package]] -name = "multiversion-macros" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d416831a7317ef4b08bee00b69cbbb9c8763da7959a7026244d6266869f9c83" -dependencies = [ - "proc-macro2", - "quote", - "rustversion", - "syn 3.0.5", -] - -[[package]] -name = "multiversion_no_op" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" - [[package]] name = "munge" version = "0.4.7" @@ -3608,7 +3468,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.13.2", + "bitflags", "cfg-if", "cfg_aliases", "libc", @@ -3645,7 +3505,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3689,9 +3549,9 @@ checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" -version = "0.1.47" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ "num-traits", ] @@ -3742,7 +3602,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.13.2", + "bitflags", ] [[package]] @@ -3772,7 +3632,7 @@ checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50" dependencies = [ "async-trait", "aws-lc-rs", - "base64 0.22.1", + "base64", "bytes", "chrono", "crc-fast", @@ -3796,7 +3656,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.20", + "thiserror 2.0.19", "tokio", "tracing", "url", @@ -3891,9 +3751,9 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "owo-colors" -version = "4.4.0" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8" +checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" [[package]] name = "papergrid" @@ -3937,9 +3797,9 @@ dependencies = [ [[package]] name = "parse-display" -version = "0.11.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78deb158fb1d73b29efb4b7e9b9860b78059c670de06bd28df8d0b458ded0eb" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" dependencies = [ "parse-display-derive", "regex", @@ -3948,9 +3808,9 @@ dependencies = [ [[package]] name = "parse-display-derive" -version = "0.11.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e95a50d1084dab562913062c4c34bb204b68fc6ec38a1395909ff5aaaf4f10a" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" dependencies = [ "proc-macro2", "quote", @@ -3997,11 +3857,11 @@ dependencies = [ [[package]] name = "pem" -version = "4.0.0" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64 0.23.1", + "base64", "serde_core", ] @@ -4013,9 +3873,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.9.1" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d45aeb61b4bf818e12d4205f2466f8c4748f85f4fce0146d1c03d69d753f0ad" +checksum = "7df728be843c7070fab6ab7c328c4e9e9d78e23bf749c0669c86ee7ebfa050a2" dependencies = [ "memchr", "ucd-trie", @@ -4023,9 +3883,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.9.1" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89cc5a242e25ed4e7704d0be240f2cfbe20a8c27e7e252d94835be93d92dc39f" +checksum = "9e2dd6fc3b26b3462ee188aac870f5a41d398f1cd5e2408d16531bd71c9591fd" dependencies = [ "pest", "pest_generator", @@ -4033,9 +3893,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.9.1" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7abf21475cc3820fe4b2ca2dc2142902f67a02189f3b5b3a229f4febc01a43e5" +checksum = "6a7a9205cfb6f596a9e8b689c0a15f9ceb7a1aafae7aaf788150ac65b29975b6" dependencies = [ "pest", "pest_meta", @@ -4046,9 +3906,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.9.1" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adba4db388f687393c18c51348d44a41d870ca9df71a2c98172ea3035dc6936e" +checksum = "85abd351c0de1e8384fc791a0737111a350394937e92b956b743dac12429f57c" dependencies = [ "pest", ] @@ -4061,7 +3921,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap 2.14.2", + "indexmap 2.14.0", ] [[package]] @@ -4109,9 +3969,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "polling" @@ -4134,34 +3994,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" dependencies = [ "cpubits", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", "universal-hash", - "zeroize", ] [[package]] name = "portable-atomic" -version = "1.15.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" - -[[package]] -name = "portable-atomic-util" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" -dependencies = [ - "portable-atomic", -] +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" [[package]] name = "potential_utf" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ - "serde_core", - "writeable", "zerovec", ] @@ -4226,7 +4074,29 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.15+spec-1.1.0", + "toml_edit 0.25.13+spec-1.1.0", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -4257,7 +4127,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 2.13.2", + "bitflags", "hex", "procfs-core", "rustix 0.38.44", @@ -4269,7 +4139,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags 2.13.2", + "bitflags", "hex", ] @@ -4287,7 +4157,7 @@ dependencies = [ "parking_lot", "procfs", "protobuf", - "thiserror 2.0.20", + "thiserror 2.0.19", ] [[package]] @@ -4298,7 +4168,7 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", "bit-vec 0.8.0", - "bitflags 2.13.2", + "bitflags", "num-traits", "rand 0.9.5", "rand_chacha", @@ -4382,22 +4252,22 @@ dependencies = [ [[package]] name = "ptr_meta" -version = "0.3.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743da816b98c921cdbe8628ef7381b76f25ecf4da599fc80aca90eae7ef70cc0" +checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" dependencies = [ "ptr_meta_derive", ] [[package]] name = "ptr_meta_derive" -version = "0.3.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c8d9ca532f185d5d4db7a7c9d51420b452168ea1c2b913953281bd6fe1fcbd0" +checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", ] [[package]] @@ -4430,7 +4300,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.20", + "thiserror 2.0.19", "tokio", "tracing", "web-time", @@ -4438,9 +4308,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.17" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "aws-lc-rs", "bytes", @@ -4453,7 +4323,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.20", + "thiserror 2.0.19", "tinyvec", "tracing", "web-time", @@ -4470,7 +4340,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4496,9 +4366,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rancor" -version = "0.1.3" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b534442d0fcdb55d66f373d9cac6d33b6293a2335bc2136dbd06ce0e87d2572" +checksum = "daff8b7b3ccf5f7ba270b3e7a0a4d4c701c5797e38dec27c7e2c3dbb830fed1c" dependencies = [ "ptr_meta", ] @@ -4578,9 +4448,9 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.14.10" +version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055" dependencies = [ "aws-lc-rs", "pem", @@ -4596,7 +4466,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.2", + "bitflags", ] [[package]] @@ -4607,27 +4477,27 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.20", + "thiserror 2.0.19", ] [[package]] name = "ref-cast" -version = "1.0.27" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.27" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -4644,9 +4514,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.18" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -4682,11 +4552,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.5" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64 0.23.1", + "base64", "bytes", "futures-core", "futures-util", @@ -4737,14 +4607,14 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.8.18" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9776093b7ca170454ab1406954f7b7d97a57c51dc6c0642957fb2ef25c2d399" +checksum = "815cc8a37159a463064825246cadb07961e25cd9885908606f6d08a98d8f8874" dependencies = [ "bytecheck", "bytes", "hashbrown 0.17.1", - "indexmap 2.14.2", + "indexmap 2.14.0", "munge", "ptr_meta", "rancor", @@ -4756,13 +4626,13 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.8.18" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c25ef604ac7dd839d44d64648952ea23c97866f124ff671b0ed2cf3ad9bb06e" +checksum = "c0ed1a78a1b19d184b0daa629dd9a024573173ec7d485b287cb369fb3607cc1c" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", ] [[package]] @@ -4771,7 +4641,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81116b9531d61eabc41aeb228e4b6b2435bcca3233b98cf3b3077d4e6e9debb3" dependencies = [ - "bitflags 2.13.2", + "bitflags", "once_cell", "serde", "serde_derive", @@ -4792,9 +4662,9 @@ dependencies = [ [[package]] name = "rstest" -version = "0.27.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "948203e6d13b83e51a90d7cf236dd58a0065f23f4b902f00f306e7eb2bd09b9c" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" dependencies = [ "futures-timer", "futures-util", @@ -4803,9 +4673,9 @@ dependencies = [ [[package]] name = "rstest_macros" -version = "0.27.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d92eaf7b6e51e12471d71ddc72608e7151573de44099aacfbb1128177c0da4" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" dependencies = [ "cfg-if", "glob", @@ -4821,12 +4691,12 @@ dependencies = [ [[package]] name = "rtoolbox" -version = "0.0.6" +version = "0.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1efe12a1469752d0e6ff5ebec0b6ef4924cc5c4c71046b0ec730040535819d" +checksum = "50a0e551c1e27e1731aba276dbeaeac73f53c7cd34d1bda485d02bd1e0f36844" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4841,9 +4711,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.43.0" +version = "1.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7653272e75dcac41dc199fbea6f5797633994fafd339943c06c9af16bf29cd3a" +checksum = "be2a24f50780bc85f09cc6ac299bdf1424302742d77221106859c9d8b102126a" dependencies = [ "arrayvec", "num-traits", @@ -4885,7 +4755,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.13.2", + "bitflags", "errno", "libc", "linux-raw-sys 0.4.15", @@ -4898,18 +4768,18 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.2", + "bitflags", "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "aws-lc-rs", "log", @@ -4961,7 +4831,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4972,9 +4842,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", @@ -5008,7 +4878,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s2-api" -version = "0.32.0" +version = "0.31.1" dependencies = [ "axum", "base64ct", @@ -5029,16 +4899,16 @@ dependencies = [ "serde", "serde_json", "strum", - "thiserror 2.0.20", + "thiserror 2.0.19", "time", "tokio-util", "utoipa", - "zstd 0.14.0", + "zstd", ] [[package]] name = "s2-cli" -version = "0.43.0" +version = "0.42.8" dependencies = [ "assert_cmd", "async-stream", @@ -5049,7 +4919,6 @@ dependencies = [ "clap", "color-print", "colored", - "compact_str", "config", "dirs", "fs2", @@ -5080,16 +4949,16 @@ dependencies = [ "serde", "serde_json", "serial_test", - "sha2", + "sha2 0.11.0", "strum", "tabled", "tempfile", "terminal_size", - "thiserror 2.0.20", + "thiserror 2.0.19", "tikv-jemallocator", "tokio", "tokio-stream", - "toml 1.1.6+spec-1.1.0", + "toml 1.1.4+spec-1.1.0", "tracing", "tracing-subscriber", "uuid", @@ -5099,7 +4968,7 @@ dependencies = [ [[package]] name = "s2-common" -version = "0.42.0" +version = "0.41.1" dependencies = [ "axum", "base64ct", @@ -5116,14 +4985,14 @@ dependencies = [ "serde", "serde_json", "strum", - "thiserror 2.0.20", + "thiserror 2.0.19", "time", "utoipa", ] [[package]] name = "s2-lite" -version = "0.43.0" +version = "0.42.8" dependencies = [ "async-stream", "async-trait", @@ -5138,7 +5007,7 @@ dependencies = [ "eyre", "futures", "http 1.5.0", - "indexmap 2.14.2", + "indexmap 2.14.0", "itertools 0.15.0", "parking_lot", "prometheus", @@ -5156,13 +5025,13 @@ dependencies = [ "serde_json", "slatedb", "strum", - "thiserror 2.0.20", + "thiserror 2.0.19", "tikv-jemallocator", "time", "tokio", "tokio-util", "tower", - "tower-http 0.7.1", + "tower-http 0.7.0", "tracing", "tracing-subscriber", "utoipa", @@ -5171,9 +5040,8 @@ dependencies = [ [[package]] name = "s2-resource-spec" -version = "0.3.0" +version = "0.2.2" dependencies = [ - "compact_str", "humantime", "s2-common", "schemars 1.2.2", @@ -5183,14 +5051,13 @@ dependencies = [ [[package]] name = "s2-sdk" -version = "0.35.0" +version = "0.34.4" dependencies = [ "assert_matches", "async-compression", "async-stream", "async-trait", "bytes", - "compact_str", "futures-core", "futures-util", "h2", @@ -5213,7 +5080,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "test-context", - "thiserror 2.0.20", + "thiserror 2.0.19", "time", "tokio", "tokio-muxt", @@ -5227,7 +5094,7 @@ dependencies = [ [[package]] name = "s2-storage" -version = "0.2.6" +version = "0.2.4" dependencies = [ "aegis", "aes-gcm", @@ -5239,17 +5106,17 @@ dependencies = [ "s2-common", "secrecy", "serde", - "thiserror 2.0.20", + "thiserror 2.0.19", ] [[package]] name = "s2-testcontainers" -version = "0.43.0" +version = "0.42.8" dependencies = [ "reqwest", "s2-sdk", "testcontainers", - "thiserror 2.0.20", + "thiserror 2.0.19", "tokio", ] @@ -5305,7 +5172,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -5325,20 +5192,20 @@ dependencies = [ [[package]] name = "secret-service" -version = "5.2.0" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5107b24b91445dd2aa449a258a1807b63240942157292354dc5bfdbeb8bc6db8" +checksum = "9a62d7f86047af0077255a29494136b9aaaf697c76ff70b8e49cded4e2623c14" dependencies = [ - "aes", + "aes 0.8.4", "cbc", "futures-util", - "getrandom 0.4.3", + "generic-array", + "getrandom 0.2.17", "hkdf", - "hybrid-array", "num", "once_cell", "serde", - "sha2", + "sha2 0.10.9", "zbus", ] @@ -5348,7 +5215,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.2", + "bitflags", "core-foundation", "core-foundation-sys", "libc", @@ -5421,7 +5288,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -5432,7 +5299,7 @@ checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -5441,7 +5308,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ - "indexmap 2.14.2", + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -5468,7 +5335,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -5503,17 +5370,16 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.23.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ - "base64 0.23.1", + "base64", "bs58", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.14.2", - "jiff", + "indexmap 2.14.0", "schemars 0.9.0", "schemars 1.2.2", "serde_core", @@ -5524,14 +5390,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.23.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ - "darling 0.24.1", + "darling 0.23.0", "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", ] [[package]] @@ -5540,7 +5406,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.14.2", + "indexmap 2.14.0", "itoa", "ryu", "serde", @@ -5569,7 +5435,7 @@ checksum = "a22144e767da4ddd8416dbf383700542ffd8a5dc493dfecedfe1fe3ad03c98ae" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -5583,6 +5449,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha2" version = "0.11.0" @@ -5590,7 +5467,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", "digest 0.11.3", ] @@ -5655,15 +5532,15 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "slatedb" -version = "0.16.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb40332f41e231926df3a0ef742c05316b43368ba4b520433d2a1eba1ab00f0f" +checksum = "35ca56b01922b15aa69fe3abb62cadc985d86032c9647e4606e211c4da751a76" dependencies = [ "async-channel", "async-trait", "atomic", "backon", - "bitflags 2.13.2", + "bitflags", "bytes", "chrono", "crc32fast", @@ -5697,14 +5574,14 @@ dependencies = [ "url", "uuid", "walkdir", - "zstd 0.13.3", + "zstd", ] [[package]] name = "slatedb-common" -version = "0.16.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b77f238f348e95e1653a5235f880ac703011b82582bd556f2aba405abf8180e" +checksum = "0aa8de522ff46a0f9b5a66f45e650d125421d4d91990933591092f9d010c40d1" dependencies = [ "chrono", "log", @@ -5718,9 +5595,9 @@ dependencies = [ [[package]] name = "slatedb-txn-obj" -version = "0.16.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad63b6c250219e26d71f497820f970274c301f896366daa56915f2a05df0e0a" +checksum = "d85fd9c0c86dd4954524fa8238d0548bd80f4a9a66983cbde3728402d339993e" dependencies = [ "async-trait", "bytes", @@ -5741,9 +5618,9 @@ checksum = "88414a5ca1f85d82cc34471e975f0f74f6aa54c40f062efa42c0080e7f763f81" [[package]] name = "smallvec" -version = "1.16.0" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" @@ -5869,9 +5746,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.5" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -5914,9 +5791,9 @@ dependencies = [ [[package]] name = "tabled" -version = "0.22.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2596a104db1900b943f97d793a6c99addca918c4525c999f751a0f17d472eb" +checksum = "b5dc662e6da844ad6e428ad16b57967c9d33c82e16bb1c258326c0c078605dff" dependencies = [ "papergrid", "tabled_derive", @@ -5925,11 +5802,12 @@ dependencies = [ [[package]] name = "tabled_derive" -version = "0.12.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dca1937322a1e892b1a65f6a6736183bb0a29d3b4234d1d53bc436dff9beb76" +checksum = "0ea5d1b13ca6cff1f9231ffd62f15eefd72543dab5e468735f1a456728a02846" dependencies = [ "heck 0.5.0", + "proc-macro-error2", "proc-macro2", "quote", "syn 2.0.119", @@ -5942,10 +5820,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5955,7 +5833,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5987,9 +5865,9 @@ dependencies = [ [[package]] name = "testcontainers" -version = "0.28.0" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2bbe381afaaa58ea610c5fc3ffb2184063a32b3e358a179f0b4865dd59934a" +checksum = "bfd5785b5483672915ed5fe3cddf9f546802779fc1eceff0a6fb7321fac81c1e" dependencies = [ "astral-tokio-tar", "async-trait", @@ -6009,7 +5887,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror 2.0.20", + "thiserror 2.0.19", "tokio", "tokio-stream", "tokio-util", @@ -6027,11 +5905,11 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.16.3" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b81c0cb5fce14f53e49c1d4da0c508334ff12040221bb8ab01b2dabd91d04b6e" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" dependencies = [ - "icu_segmenter", + "unicode-linebreak", "unicode-width 0.2.2", ] @@ -6046,11 +5924,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.20" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" dependencies = [ - "thiserror-impl 2.0.20", + "thiserror-impl 2.0.19", ] [[package]] @@ -6066,13 +5944,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.20" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -6106,9 +5984,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.55" +version = "0.3.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" dependencies = [ "deranged", "num-conv", @@ -6145,20 +6023,19 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.4" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", - "serde_core", "zerovec", ] [[package]] name = "tinyvec" -version = "1.13.2" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" dependencies = [ "tinyvec_macros", ] @@ -6194,7 +6071,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -6209,9 +6086,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.5" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ "rustls", "tokio", @@ -6280,11 +6157,11 @@ dependencies = [ [[package]] name = "toml" -version = "1.1.6+spec-1.1.0" +version = "1.1.4+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "920602543f0911ab71da12c50d59701da54c196d1a2bf5cb4b75667f137a406a" +checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" dependencies = [ - "indexmap 2.14.2", + "indexmap 2.14.0", "serde_core", "serde_spanned 1.1.1", "toml_datetime 1.1.1+spec-1.1.0", @@ -6317,7 +6194,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.14.2", + "indexmap 2.14.0", "serde", "serde_spanned 0.6.9", "toml_datetime 0.6.11", @@ -6327,11 +6204,11 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.15+spec-1.1.0" +version = "0.25.13+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" dependencies = [ - "indexmap 2.14.2", + "indexmap 2.14.0", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "winnow 1.0.4", @@ -6366,7 +6243,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", - "base64 0.22.1", + "base64", "bytes", "h2", "http 1.5.0", @@ -6406,7 +6283,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "indexmap 2.14.2", + "indexmap 2.14.0", "pin-project-lite", "slab", "sync_wrapper", @@ -6423,7 +6300,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.13.2", + "bitflags", "bytes", "futures-util", "http 1.5.0", @@ -6437,12 +6314,12 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.7.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c" +checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" dependencies = [ "async-compression", - "bitflags 2.13.2", + "bitflags", "bytes", "futures-core", "http 1.5.0", @@ -6539,9 +6416,12 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twox-hash" -version = "2.1.4" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" +dependencies = [ + "rand 0.10.2", +] [[package]] name = "typed-path" @@ -6610,6 +6490,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -6662,6 +6548,33 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "ureq-proto", + "utf8-zero", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64", + "http 1.5.0", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -6681,6 +6594,12 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -6696,9 +6615,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "utoipa" version = "5.4.0" -source = "git+https://github.com/infiniteregrets/utoipa?rev=82bcb28a792ba9a0d29963827ec473823099fc94#82bcb28a792ba9a0d29963827ec473823099fc94" +source = "git+https://github.com/infiniteregrets/utoipa?rev=9cd181d40ac0a50551ebe1995a4d73e8685890d6#9cd181d40ac0a50551ebe1995a4d73e8685890d6" dependencies = [ - "indexmap 2.14.2", + "indexmap 2.14.0", "serde", "serde_json", "utoipa-gen", @@ -6707,7 +6626,7 @@ dependencies = [ [[package]] name = "utoipa-gen" version = "5.4.0" -source = "git+https://github.com/infiniteregrets/utoipa?rev=82bcb28a792ba9a0d29963827ec473823099fc94#82bcb28a792ba9a0d29963827ec473823099fc94" +source = "git+https://github.com/infiniteregrets/utoipa?rev=9cd181d40ac0a50551ebe1995a4d73e8685890d6#9cd181d40ac0a50551ebe1995a4d73e8685890d6" dependencies = [ "proc-macro2", "quote", @@ -6716,9 +6635,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.26.1" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -6790,9 +6709,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.128" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -6803,9 +6722,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.78" +version = "0.4.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" dependencies = [ "js-sys", "wasm-bindgen", @@ -6813,9 +6732,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.128" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6823,22 +6742,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.128" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.128" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] @@ -6858,9 +6777,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.105" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" dependencies = [ "js-sys", "wasm-bindgen", @@ -6907,7 +6826,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -7203,9 +7122,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.4" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "x509-parser" @@ -7221,7 +7140,7 @@ dependencies = [ "nom", "oid-registry", "rusticata-macros", - "thiserror 2.0.20", + "thiserror 2.0.19", "time", ] @@ -7249,9 +7168,9 @@ checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "yaml-rust2" -version = "0.11.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b36710ce3a279cfce8465dbab826f161675a262950b922cb2c3663852dfe9eb0" +checksum = "631a50d867fafb7093e709d75aaee9e0e0d5deb934021fcea25ac2fe09edc51e" dependencies = [ "arraydeque", "encoding_rs", @@ -7299,9 +7218,9 @@ dependencies = [ [[package]] name = "zbus" -version = "5.19.0" +version = "5.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db4be7c075cb421e4b7ee645541604239bd243ba7c357511f4ff3a74b555907" +checksum = "fe18fb60dc696039e738717b76eaea21e7a4489bbb1885020b43c94236d7e98a" dependencies = [ "async-broadcast", "async-executor", @@ -7334,9 +7253,9 @@ dependencies = [ [[package]] name = "zbus-secret-service-keyring-store" -version = "1.0.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74801d001b9e7729adb4f1825b67b398185fed424749aa3d8bacf70417137d9a" +checksum = "4ccede190ba363386a24e8021c7f3848393976609ec9f5d1f8c6c09ef37075b4" dependencies = [ "keyring-core", "secret-service", @@ -7345,14 +7264,14 @@ dependencies = [ [[package]] name = "zbus_macros" -version = "5.19.0" +version = "5.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2990635d09ade6df1868f72f8cac69a876a90981e8bd3c40b1be413f8dc88f40" +checksum = "fe96480bed92df2b442a1a30df364e12d08eed03aeb061f2b8dc6afb2be91119" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", "zbus_names", "zvariant", "zvariant_utils", @@ -7369,29 +7288,20 @@ dependencies = [ "zvariant", ] -[[package]] -name = "zcheapstr" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1afec51604565183aeb5c54c20aeab286120d4e4460f7f76e3e8bb8c0d99473" -dependencies = [ - "serde", -] - [[package]] name = "zerocopy" -version = "0.8.57" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.57" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", @@ -7427,23 +7337,21 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.5" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", "zerofrom", - "zerovec", ] [[package]] name = "zerovec" -version = "0.11.8" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ - "serde", "yoke", "zerofrom", "zerovec-derive", @@ -7451,13 +7359,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.6" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", ] [[package]] @@ -7468,7 +7376,7 @@ checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" dependencies = [ "crc32fast", "flate2", - "indexmap 2.14.2", + "indexmap 2.14.0", "memchr", "typed-path", "zopfli", @@ -7476,9 +7384,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.6.7" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" +checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5" [[package]] name = "zmij" @@ -7504,41 +7412,23 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe 7.3.0", -] - -[[package]] -name = "zstd" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e" -dependencies = [ - "zstd-safe 8.0.0", + "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.3.0" +version = "7.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-safe" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.1.0+zstd.1.5.7" +version = "2.0.16+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" dependencies = [ "cc", "pkg-config", @@ -7546,41 +7436,40 @@ dependencies = [ [[package]] name = "zvariant" -version = "5.15.0" +version = "5.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1d34c27cc6cdd1f458427519dd6b8612f7b7e3f7b9a0b2355d041dda9869147" +checksum = "bee2a0bcd2a907786a456fff45aaaaf54c9ba5f50b71ae9ec1a4edd200c94911" dependencies = [ "endi", "enumflags2", "serde", "winnow 1.0.4", - "zcheapstr", "zvariant_derive", "zvariant_utils", ] [[package]] name = "zvariant_derive" -version = "5.15.0" +version = "5.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "864155e69b4352db0c7f374917bf45d1e0c8d17659c8b3dbf9795f3673f8c497" +checksum = "38a708216a18780796770bfe3f4739c7c83a3e8f789b755534bbbc06e4e23e12" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", "zvariant_utils", ] [[package]] name = "zvariant_utils" -version = "4.2.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad0294361a320b694a328460dc73add56c306150f5cb6bfafc44446120008a3" +checksum = "90cb9383f9b45290407a1258b202d3f8f01db719eb60b4e4055c6375af4fc7c7" dependencies = [ "proc-macro2", "quote", "serde", - "syn 3.0.5", + "syn 2.0.119", "winnow 1.0.4", ] diff --git a/Cargo.toml b/Cargo.toml index a76dfc09..a092a9b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,14 +29,14 @@ axum-server = "0.8" base64ct = "1.8" blake3 = "1.8" bytes = "1" -bytesize = "2.7" +bytesize = "2.6" clap = "4.6" color-print = "0.3" colored = "3.1" compact_str = "0.10" config = "0.15" dashmap = "6.2" -dirs = "7.0" +dirs = "6.0" enumset = "1.1" eyre = "0.6" flate2 = "1.1" @@ -49,8 +49,8 @@ humantime = "2.4" indexmap = "2.14" indicatif = "0.18" itertools = "0.15" -json_to_table = "0.14" -keyring = "4.2" +json_to_table = "0.13" +keyring = "4.1" miette = "7.6" mime = "0.3" parking_lot = "0.12" @@ -63,24 +63,24 @@ rcgen = { version = "0.14", default-features = false, features = ["crypto", "pem reqwest = { version = "0.13", default-features = false, features = ["rustls"] } rkyv = "0.8" rpassword = "7.5" -rstest = "0.27" +rstest = "0.26" rustls = { version = "0.23", default-features = false, features = ["logging", "std", "tls12"] } -s2-api = { path = "api", version = "0.32" } -s2-common = { path = "common", version = "0.42" } -s2-lite = { path = "lite", version = "0.43" } -s2-resource-spec = { path = "resource-spec", version = "0.3" } -s2-sdk = { path = "sdk", version = "0.35" } +s2-api = { path = "api", version = "0.31" } +s2-common = { path = "common", version = "0.41" } +s2-lite = { path = "lite", version = "0.42" } +s2-resource-spec = { path = "resource-spec", version = "0.2" } +s2-sdk = { path = "sdk", version = "0.34" } s2-storage = { path = "storage", version = "0.2" } schemars = "1.2" secrecy = "0.10.3" semver = "1.0" serde = "1.0" serde_json = "1.0" -slatedb = "0.16.0" +slatedb = "0.15.0" strum = "0.28" -tabled = "0.22" +tabled = "0.21" tempfile = "3.27" -testcontainers = "0.28" +testcontainers = "0.27" thiserror = "2.0" tikv-jemallocator = { version = "0.7", features = ["unprefixed_malloc_on_supported_platforms"] } time = "0.3" @@ -92,12 +92,12 @@ tower-http = "0.7" tracing = "0.1" tracing-subscriber = "0.3" utoipa = "=5.4" -uuid = "1.26" +uuid = "1.24" xxhash-rust = "0.8" -zstd = "0.14" +zstd = "0.13" [patch.crates-io] -utoipa = { git = "https://github.com/infiniteregrets/utoipa", rev = "82bcb28a792ba9a0d29963827ec473823099fc94" } +utoipa = { git = "https://github.com/infiniteregrets/utoipa", rev = "9cd181d40ac0a50551ebe1995a4d73e8685890d6" } [profile.dev] panic = "abort" diff --git a/README.md b/README.md index 6e3acc97..9c3eb14e 100644 --- a/README.md +++ b/README.md @@ -194,34 +194,6 @@ nc starwars.s2.dev 23 | s2 append s2://liteness/starwars Deploy `s2-lite` to Kubernetes using Helm. See the [Helm chart documentation](charts/s2-lite-helm/README.md) for installation instructions and configuration options. -### Storage - -Lite persists to an S3-compatible bucket or a local directory, or runs in-memory when neither is given. -The write-ahead log (WAL) shares the main store by default; it can be placed in a separate bucket or -directory to isolate WAL latency from flushes and compaction. `--path` applies to both stores. - -| Setting | Main store | WAL store | -| --- | --- | --- | -| S3 bucket | `--bucket` | `--wal-bucket` / `S2LITE_WAL_BUCKET` | -| Local directory | `--local-root` | `--wal-local-root` / `S2LITE_WAL_LOCAL_ROOT` | -| S3 endpoint | `AWS_ENDPOINT_URL_S3` | `S2LITE_WAL_AWS_ENDPOINT_URL_S3` | -| AWS region | `AWS_REGION` | `S2LITE_WAL_AWS_REGION` | -| Static credentials | `AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY` | `S2LITE_WAL_AWS_ACCESS_KEY_ID` + `S2LITE_WAL_AWS_SECRET_ACCESS_KEY` | -| Session token | `AWS_SESSION_TOKEN` | `S2LITE_WAL_AWS_SESSION_TOKEN` | - -Without static credentials, the standard AWS credential chain (profile, instance role, etc.) is used. -The WAL bucket inherits the main store's S3 settings; each `S2LITE_WAL_AWS_*` variable overrides just -that setting, except that a WAL key pair replaces the main credentials (and session token) as a set. -A WAL store requires a persistent main store. - -```bash -# LSM in S3, WAL on local disk -s2 lite --bucket my-bucket --wal-local-root /data/wal - -# LSM and WAL in separate buckets -s2 lite --bucket my-bucket --wal-bucket my-wal-bucket -``` - ### Monitoring `/health` will return 200 on success for readiness and liveness checks @@ -237,7 +209,7 @@ s2 lite --bucket my-bucket --wal-bucket my-wal-bucket Use `SL8_` prefixed environment variables, e.g.: ```bash -# Defaults to 50ms for S3, 5ms otherwise; follows the WAL store when set +# Defaults to 50ms for remote bucket / 5ms in-memory SL8_FLUSH_INTERVAL=10ms ``` diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index 55052994..4c989e96 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -2,51 +2,6 @@ All notable changes to this project will be documented in this file. -## [0.32.0] - 2026-09-25 - -### Features - -- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) - -### Bug Fixes - -- Skip s2s response compression the client refused with q=0 ([#781](https://github.com/s2-streamstore/s2/issues/781)) - - - -## [0.31.5] - 2026-09-22 - -### Bug Fixes - -- Preserve SSE read budgets across reconnects ([#727](https://github.com/s2-streamstore/s2/issues/727)) - - - -## [0.31.4] - 2026-09-16 - -### Miscellaneous Tasks - -- Sync specs submodule ([#739](https://github.com/s2-streamstore/s2/issues/739)) - - - -## [0.31.3] - 2026-09-11 - -### Features - -- `s2-stream-config` header for auto-created streams ([#718](https://github.com/s2-streamstore/s2/issues/718)) - - - -## [0.31.2] - 2026-09-10 - -### Bug Fixes - -- Reject NUL bytes in stream names and access token IDs ([#728](https://github.com/s2-streamstore/s2/issues/728)) -- Switch JSON extractor from sonic-rs back to serde_json ([#729](https://github.com/s2-streamstore/s2/issues/729)) - - - ## [0.31.1] - 2026-08-13 ### Features diff --git a/api/Cargo.toml b/api/Cargo.toml index 35cd9cee..691ccc7a 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-api" -version = "0.32.0" +version = "0.31.1" description = "API types for S2, the durable streams API" edition.workspace = true license.workspace = true diff --git a/api/specs b/api/specs index edaa1fbc..158dede9 160000 --- a/api/specs +++ b/api/specs @@ -1 +1 @@ -Subproject commit edaa1fbcb2507362d6c680c66cdc88660e73e036 +Subproject commit 158dede97299fd4f7a86cd842f19157843e57e7d diff --git a/api/src/data.rs b/api/src/data.rs index 9e53a92f..f9fc4fc7 100644 --- a/api/src/data.rs +++ b/api/src/data.rs @@ -96,25 +96,6 @@ pub struct S2FormatHeader { pub s2_format: Format, } -#[rustfmt::skip] -#[derive(Debug)] -#[cfg_attr(feature = "utoipa", derive(utoipa::IntoParams))] -#[cfg_attr(feature = "utoipa", into_params(parameter_in = Header))] -pub struct S2StreamConfigHeader { - /// JSON-encoded `StreamConfig` to apply if the stream is created on append or read. - /// Unset fields inherit the basin's default stream configuration. - /// Ignored if the stream already exists. - /// Compact JSON is preferred. - #[cfg_attr(feature = "utoipa", param( - required = false, - rename = "s2-stream-config", - content_type = "application/json", - value_type = crate::v1::config::StreamConfig, - example = json!({"retention_policy":{"age":3600},"delete_on_empty":{"min_age_secs":300}}), - ))] - pub s2_stream_config: String, -} - #[rustfmt::skip] #[derive(Debug)] #[cfg_attr(feature = "utoipa", derive(utoipa::IntoParams))] @@ -466,7 +447,7 @@ pub mod extract { // StreamReconfiguration: exercises Maybe in all three states use s2_common::maybe::Maybe; - use crate::v1::config::{TimestampingMode, TimestampingReconfiguration}; + use crate::v1::config::{StorageClass, TimestampingMode, TimestampingReconfiguration}; // All fields unspecified (empty JSON object) assert_roundtrip(&StreamReconfiguration { @@ -477,7 +458,7 @@ pub mod extract { }); // Mix of specified-null and specified-value assert_roundtrip(&StreamReconfiguration { - storage_class: Maybe::Specified(Some("express".into())), + storage_class: Maybe::Specified(Some(StorageClass::Express)), retention_policy: Maybe::Specified(None), timestamping: Maybe::Specified(Some(TimestampingReconfiguration { mode: Maybe::Specified(Some(TimestampingMode::ClientRequire)), diff --git a/api/src/v1/config.rs b/api/src/v1/config.rs index 671c0cb8..fc523439 100644 --- a/api/src/v1/config.rs +++ b/api/src/v1/config.rs @@ -1,10 +1,37 @@ -use std::{str::FromStr, time::Duration}; +use std::time::Duration; -use compact_str::CompactString; -use http::{HeaderName, HeaderValue}; -use s2_common::{http::ParseableHeader, maybe::Maybe}; +use s2_common::maybe::Maybe; use serde::{Deserialize, Serialize}; +#[rustfmt::skip] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] +#[serde(rename_all = "kebab-case")] +pub enum StorageClass { + /// Append tail latency under 400 milliseconds with s2.dev. + Standard, + /// Append tail latency under 40 milliseconds with s2.dev. + Express, +} + +impl From for s2_common::config::StorageClass { + fn from(value: StorageClass) -> Self { + match value { + StorageClass::Express => Self::Express, + StorageClass::Standard => Self::Standard, + } + } +} + +impl From for StorageClass { + fn from(value: s2_common::config::StorageClass) -> Self { + match value { + s2_common::config::StorageClass::Express => Self::Express, + s2_common::config::StorageClass::Standard => Self::Standard, + } + } +} + #[rustfmt::skip] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] @@ -271,9 +298,8 @@ impl From for EncryptionAlgorithm { #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] pub struct StreamConfig { - /// [Storage class](https://s2.dev/docs/storage-classes) for recent writes. - #[cfg_attr(feature = "utoipa", schema(value_type = Option))] - pub storage_class: Option, + /// Storage class for recent writes. + pub storage_class: Option, /// Retention policy for the stream. /// If unspecified, the default is to retain records for 7 days. pub retention_policy: Option, @@ -294,7 +320,7 @@ impl StreamConfig { } = config; let config = StreamConfig { - storage_class, + storage_class: storage_class.map(Into::into), retention_policy: retention_policy.map(Into::into), timestamping: TimestampingConfig::to_opt(timestamping), delete_on_empty: DeleteOnEmptyConfig::to_opt(delete_on_empty), @@ -305,12 +331,6 @@ impl StreamConfig { Some(config) } } - - /// Encode as compact JSON for the `s2-stream-config` header. - pub fn to_header_value(&self) -> HeaderValue { - let json = serde_json::to_string(self).expect("StreamConfig serializes to JSON"); - HeaderValue::from_str(&json).expect("compact JSON of StreamConfig is a valid header value") - } } impl From for StreamConfig { @@ -323,7 +343,7 @@ impl From for StreamConfig { } = value; Self { - storage_class, + storage_class: Some(storage_class.into()), retention_policy: Some(retention_policy.into()), timestamping: Some(timestamping.into()), delete_on_empty: Some(delete_on_empty.into()), @@ -331,30 +351,6 @@ impl From for StreamConfig { } } -pub static STREAM_CONFIG_HEADER: HeaderName = HeaderName::from_static("s2-stream-config"); - -/// Value of the `s2-stream-config` header: a JSON-encoded [`StreamConfig`] to apply over the -/// basin's default stream config if the stream is created on append or read. Ignored if the -/// stream already exists. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct StreamConfigHeader(pub s2_common::config::OptionalStreamConfig); - -impl FromStr for StreamConfigHeader { - type Err = s2_common::ValidationError; - - fn from_str(s: &str) -> Result { - let config: StreamConfig = - serde_json::from_str(s).map_err(|e| format!("invalid JSON: {e}"))?; - Ok(Self(config.try_into()?)) - } -} - -impl ParseableHeader for StreamConfigHeader { - fn name() -> &'static HeaderName { - &STREAM_CONFIG_HEADER - } -} - impl TryFrom for s2_common::config::OptionalStreamConfig { type Error = s2_common::ValidationError; @@ -372,7 +368,7 @@ impl TryFrom for s2_common::config::OptionalStreamConfig { }; let config = Self { - storage_class, + storage_class: storage_class.map(Into::into), retention_policy, timestamping: timestamping.map(Into::into).unwrap_or_default(), delete_on_empty: delete_on_empty.map(Into::into).unwrap_or_default(), @@ -386,10 +382,10 @@ impl TryFrom for s2_common::config::OptionalStreamConfig { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] pub struct StreamReconfiguration { - /// [Storage class](https://s2.dev/docs/storage-classes) for recent writes. + /// Storage class for recent writes. #[serde(default, skip_serializing_if = "Maybe::is_unspecified")] - #[cfg_attr(feature = "utoipa", schema(value_type = Option))] - pub storage_class: Maybe>, + #[cfg_attr(feature = "utoipa", schema(value_type = Option))] + pub storage_class: Maybe>, /// Retention policy for the stream. /// If unspecified, the default is to retain records for 7 days. #[serde(default, skip_serializing_if = "Maybe::is_unspecified")] @@ -417,7 +413,7 @@ impl TryFrom for s2_common::config::StreamReconfiguration } = value; Ok(Self { - storage_class, + storage_class: storage_class.map_opt(Into::into), retention_policy: retention_policy.try_map_opt(TryInto::try_into)?, timestamping: timestamping.map_opt(Into::into), delete_on_empty: delete_on_empty.map_opt(Into::into), @@ -435,7 +431,7 @@ impl From for StreamReconfiguration { } = value; Self { - storage_class, + storage_class: storage_class.map_opt(Into::into), retention_policy: retention_policy.map_opt(Into::into), timestamping: timestamping.map_opt(Into::into), delete_on_empty: delete_on_empty.map_opt(Into::into), @@ -570,8 +566,8 @@ mod tests { use super::*; - fn gen_storage_class() -> impl Strategy { - "[a-z][a-z0-9-]{0,30}".prop_map(CompactString::from) + fn gen_storage_class() -> impl Strategy { + prop_oneof![Just(StorageClass::Standard), Just(StorageClass::Express)] } fn gen_timestamping_mode() -> impl Strategy { @@ -728,7 +724,7 @@ mod tests { ) .prop_map(|(sc, rp, ts_mode, ts_uncapped, doe)| { s2_common::config::OptionalStreamConfig { - storage_class: sc, + storage_class: sc.map(Into::into), retention_policy: rp.map(|rp| match rp { RetentionPolicy::Age(secs) => { s2_common::config::RetentionPolicy::Age(Duration::from_secs(secs)) @@ -800,7 +796,7 @@ mod tests { prop_assert_eq!( merged.storage_class, - stream.storage_class.or(basin.storage_class) + stream.storage_class.or(basin.storage_class).unwrap_or_default() ); prop_assert_eq!( merged.retention_policy, @@ -856,7 +852,7 @@ mod tests { new_rp_secs in 1u64..u64::MAX, ) { let reconfig = s2_common::config::StreamReconfiguration { - storage_class: Maybe::Specified(Some(new_sc.clone())), + storage_class: Maybe::Specified(Some(new_sc.into())), retention_policy: Maybe::Specified(Some( s2_common::config::RetentionPolicy::Age(Duration::from_secs(new_rp_secs)) )), @@ -864,7 +860,7 @@ mod tests { }; let result = base.reconfigure(reconfig); - prop_assert_eq!(result.storage_class, Some(new_sc)); + prop_assert_eq!(result.storage_class, Some(new_sc.into())); prop_assert_eq!( result.retention_policy, Some(s2_common::config::RetentionPolicy::Age(Duration::from_secs(new_rp_secs))) @@ -879,7 +875,7 @@ mod tests { ) { // non-default storage class -> Some let internal = s2_common::config::OptionalStreamConfig { - storage_class: Some(sc), + storage_class: Some(sc.into()), ..Default::default() }; prop_assert!(StreamConfig::to_opt(internal).is_some()); @@ -927,7 +923,7 @@ mod tests { ) { let base = s2_common::config::BasinConfig { default_stream_config: s2_common::config::OptionalStreamConfig { - storage_class: base_sc, + storage_class: base_sc.map(Into::into), ..Default::default() }, stream_cipher: base_algorithm.map(Into::into), @@ -958,7 +954,7 @@ mod tests { let reconfig = s2_common::config::BasinReconfiguration { default_stream_config: Maybe::Specified(Some(s2_common::config::StreamReconfiguration { - storage_class: Maybe::Specified(Some(new_sc.clone())), + storage_class: Maybe::Specified(Some(new_sc.into())), ..Default::default() })), stream_cipher: Maybe::Specified(Some(new_algorithm.into())), @@ -967,7 +963,7 @@ mod tests { }; let result = base.reconfigure(reconfig); - prop_assert_eq!(result.default_stream_config.storage_class, Some(new_sc)); + prop_assert_eq!(result.default_stream_config.storage_class, Some(new_sc.into())); prop_assert_eq!(result.stream_cipher, Some(new_algorithm.into())); prop_assert_eq!(result.create_stream_on_append, new_on_append); } @@ -1061,67 +1057,4 @@ mod tests { "delete_on_empty.min_age should be None" ); } - - #[test] - fn stream_config_header_parses_and_validates() { - let header: StreamConfigHeader = - r#"{"retention_policy":{"age":3600},"delete_on_empty":{"min_age_secs":300}}"# - .parse() - .unwrap(); - assert_eq!( - header.0, - s2_common::config::OptionalStreamConfig { - retention_policy: Some(s2_common::config::RetentionPolicy::Age( - Duration::from_secs(3600) - )), - delete_on_empty: s2_common::config::OptionalDeleteOnEmptyConfig { - min_age: Some(Duration::from_secs(300)), - }, - ..Default::default() - } - ); - - for spaced in [ - r#"{ "retention_policy": { "age": 3600 }, "delete_on_empty": { "min_age_secs": 300 } }"#, - "{\t\"delete_on_empty\":\t{\"min_age_secs\":\t300},\t\"retention_policy\":\t{\"age\":\t3600}\t}", - " {\"retention_policy\":{\"age\":3600},\"delete_on_empty\":{\"min_age_secs\":300}} ", - ] { - let parsed: StreamConfigHeader = spaced.parse().unwrap(); - assert_eq!(parsed, header, "{spaced:?}"); - } - - let empty: StreamConfigHeader = "{}".parse().unwrap(); - assert_eq!(empty.0, Default::default()); - - let invalid_json = "not json".parse::().unwrap_err(); - assert!(invalid_json.to_string().contains("invalid JSON")); - - let invalid_age = - r#"{"retention_policy":{"age":0}}"#.parse::().unwrap_err(); - assert!( - invalid_age - .to_string() - .contains("age must be greater than 0 seconds"), - "{invalid_age}" - ); - } - - #[test] - fn stream_config_header_value_roundtrips() { - let config = StreamConfig { - storage_class: Some("express".into()), - retention_policy: Some(RetentionPolicy::Infinite(InfiniteRetention {})), - timestamping: Some(TimestampingConfig { - mode: Some(TimestampingMode::ClientRequire), - uncapped: Some(true), - }), - delete_on_empty: Some(DeleteOnEmptyConfig { min_age_secs: 60 }), - }; - let value = config.to_header_value(); - let parsed: StreamConfigHeader = value.to_str().unwrap().parse().unwrap(); - assert_eq!( - parsed.0, - s2_common::config::OptionalStreamConfig::try_from(config).unwrap() - ); - } } diff --git a/api/src/v1/location.rs b/api/src/v1/location.rs index aacb78a4..408486e7 100644 --- a/api/src/v1/location.rs +++ b/api/src/v1/location.rs @@ -1,4 +1,3 @@ -use compact_str::CompactString; use s2_common::{self, location::LocationName}; use serde::{Deserialize, Serialize}; @@ -10,31 +9,13 @@ pub struct LocationInfo { pub name: LocationName, /// Location represents a private placement, limited by account. pub is_private: bool, - /// [Storage classes](https://s2.dev/docs/storage-classes) available to the account in this location. - #[serde(default, skip_serializing_if = "Option::is_none")] - #[cfg_attr(feature = "utoipa", schema(value_type = Option>))] - pub storage_classes: Option>, - /// Default [storage class](https://s2.dev/docs/storage-classes) for this location. - #[serde(default, skip_serializing_if = "Option::is_none")] - #[cfg_attr(feature = "utoipa", schema(value_type = Option))] - pub default_storage_class: Option, } impl From for LocationInfo { fn from(value: s2_common::location::LocationInfo) -> Self { - let s2_common::location::LocationInfo { - name, - is_private, - storage_classes, - default_storage_class, - } = value; + let s2_common::location::LocationInfo { name, is_private } = value; - Self { - name, - is_private, - storage_classes: Some(storage_classes), - default_storage_class: Some(default_storage_class), - } + Self { name, is_private } } } diff --git a/api/src/v1/stream/extract.rs b/api/src/v1/stream/extract.rs index deae94fb..6d508d45 100644 --- a/api/src/v1/stream/extract.rs +++ b/api/src/v1/stream/extract.rs @@ -17,7 +17,7 @@ use crate::{ extract::{JsonExtractionRejection, ProtoRejection}, }, mime::JsonOrProto, - v1::{config::StreamConfigHeader, stream::sse::LastEventId}, + v1::stream::sse::LastEventId, }; #[derive(Debug, thiserror::Error)] @@ -54,9 +54,6 @@ where async fn from_request(req: Request, state: &S) -> Result { let content_type = crate::mime::content_type(req.headers()); let encryption_key = parse_header_opt::(req.headers())?; - let create_stream_config_patch = parse_header_opt::(req.headers())? - .map(|header| header.0) - .unwrap_or_default(); if content_type.as_ref().is_some_and(crate::mime::is_s2s_proto) { let response_compression = @@ -91,7 +88,6 @@ where return Ok(Self::S2s { encryption_key, - create_stream_config_patch, inputs: Box::pin(inputs), response_compression, }); @@ -121,7 +117,6 @@ where Ok(Self::Unary { encryption_key, - create_stream_config_patch, input, response_mime, }) @@ -137,16 +132,12 @@ where async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { let content_type = crate::mime::content_type(&parts.headers); let encryption_key = parse_header_opt::(&parts.headers)?; - let create_stream_config_patch = parse_header_opt::(&parts.headers)? - .map(|header| header.0) - .unwrap_or_default(); if content_type.as_ref().is_some_and(crate::mime::is_s2s_proto) { let response_compression = s2s::CompressionAlgorithm::from_accept_encoding(&parts.headers); return Ok(Self::S2s { encryption_key, - create_stream_config_patch, response_compression, }); } @@ -159,7 +150,6 @@ where let last_event_id = parse_header_opt::(&parts.headers)?; return Ok(Self::EventStream { encryption_key, - create_stream_config_patch, format, last_event_id, }); @@ -172,7 +162,6 @@ where Ok(Self::Unary { encryption_key, - create_stream_config_patch, format, response_mime, }) diff --git a/api/src/v1/stream/mod.rs b/api/src/v1/stream/mod.rs index 2d8d66e8..255879cd 100644 --- a/api/src/v1/stream/mod.rs +++ b/api/src/v1/stream/mod.rs @@ -11,7 +11,6 @@ use std::time::Duration; use futures_core::stream::BoxStream; use itertools::Itertools as _; use s2_common::{ - config::OptionalStreamConfig, encryption::EncryptionKey, record, stream::{StreamName, StreamNamePrefix, StreamNameStartAfter}, @@ -211,24 +210,18 @@ pub enum ReadRequest { /// Unary Unary { encryption_key: Option, - /// Parsed `s2-stream-config` header; empty if absent. - create_stream_config_patch: OptionalStreamConfig, format: Format, response_mime: JsonOrProto, }, /// Server-Sent Events streaming response EventStream { encryption_key: Option, - /// Parsed `s2-stream-config` header; empty if absent. - create_stream_config_patch: OptionalStreamConfig, format: Format, last_event_id: Option, }, /// S2S streaming response S2s { encryption_key: Option, - /// Parsed `s2-stream-config` header; empty if absent. - create_stream_config_patch: OptionalStreamConfig, response_compression: s2s::CompressionAlgorithm, }, } @@ -237,16 +230,12 @@ pub enum AppendRequest { /// Unary Unary { encryption_key: Option, - /// Parsed `s2-stream-config` header; empty if absent. - create_stream_config_patch: OptionalStreamConfig, input: s2_common::stream::AppendInput, response_mime: JsonOrProto, }, /// S2S bi-directional streaming S2s { encryption_key: Option, - /// Parsed `s2-stream-config` header; empty if absent. - create_stream_config_patch: OptionalStreamConfig, inputs: BoxStream<'static, Result>, response_compression: s2s::CompressionAlgorithm, }, @@ -257,25 +246,21 @@ impl std::fmt::Debug for AppendRequest { match self { AppendRequest::Unary { encryption_key, - create_stream_config_patch, input, response_mime: response, } => f .debug_struct("AppendRequest::Unary") .field("encryption_key", encryption_key) - .field("create_stream_config_patch", create_stream_config_patch) .field("input", input) .field("response", response) .finish(), AppendRequest::S2s { encryption_key, - create_stream_config_patch, response_compression, .. } => f .debug_struct("AppendRequest::S2s") .field("encryption_key", encryption_key) - .field("create_stream_config_patch", create_stream_config_patch) .field("response_compression", response_compression) .finish(), } diff --git a/api/src/v1/stream/s2s.rs b/api/src/v1/stream/s2s.rs index ae6c6304..517745d3 100644 --- a/api/src/v1/stream/s2s.rs +++ b/api/src/v1/stream/s2s.rs @@ -71,11 +71,7 @@ impl CompressionAlgorithm { for header_value in headers.get_all(http::header::ACCEPT_ENCODING) { if let Ok(value) = header_value.to_str() { for encoding in value.split(',') { - let mut parts = encoding.split(';'); - let encoding = parts.next().unwrap_or("").trim(); - if parts.any(is_zero_qvalue) { - continue; - } + let encoding = encoding.trim().split(';').next().unwrap_or("").trim(); if encoding.eq_ignore_ascii_case("zstd") { return Self::Zstd; } else if encoding.eq_ignore_ascii_case("gzip") { @@ -88,15 +84,6 @@ impl CompressionAlgorithm { } } -/// Whether an `Accept-Encoding` parameter is a `q=0` weight, which marks the coding as -/// "not acceptable" (RFC 9110 §12.4.2). -fn is_zero_qvalue(param: &str) -> bool { - let Some((name, value)) = param.split_once('=') else { - return false; - }; - name.trim().eq_ignore_ascii_case("q") && value.trim().parse::().is_ok_and(|q| q == 0.0) -} - #[derive(Debug, Clone, PartialEq, Eq)] pub struct CompressedData { compression: CompressionAlgorithm, @@ -591,26 +578,6 @@ mod test { assert_eq!(algo, CompressionAlgorithm::Gzip); } - #[rstest::rstest] - #[case("zstd;q=0, gzip", CompressionAlgorithm::Gzip)] - #[case("zstd; q=0.0, gzip;q=0.5", CompressionAlgorithm::Gzip)] - #[case("gzip;q=0", CompressionAlgorithm::None)] - #[case("gzip;Q=0.000, zstd;q=0", CompressionAlgorithm::None)] - #[case("zstd;q=0.001", CompressionAlgorithm::Zstd)] - fn from_accept_encoding_skips_refused_codings( - #[case] accept_encoding: &'static str, - #[case] expected: CompressionAlgorithm, - ) { - let mut headers = http::HeaderMap::new(); - headers.insert( - http::header::ACCEPT_ENCODING, - HeaderValue::from_static(accept_encoding), - ); - - let algo = CompressionAlgorithm::from_accept_encoding(&headers); - assert_eq!(algo, expected); - } - #[test] fn from_accept_encoding_defaults_to_none() { let headers = http::HeaderMap::new(); diff --git a/api/src/v1/stream/sse.rs b/api/src/v1/stream/sse.rs index 7fad5d0f..f6c5b955 100644 --- a/api/src/v1/stream/sse.rs +++ b/api/src/v1/stream/sse.rs @@ -9,11 +9,8 @@ static LAST_EVENT_ID_HEADER: http::HeaderName = http::HeaderName::from_static("l #[derive(Debug, Clone, Copy)] pub struct LastEventId { - /// Sequence number of the last delivered record. pub seq_num: u64, - /// Total records delivered. pub count: usize, - /// Total metered bytes delivered. pub bytes: usize, } diff --git a/charts/s2-lite-helm/Chart.yaml b/charts/s2-lite-helm/Chart.yaml index 953a1674..35c7c06e 100644 --- a/charts/s2-lite-helm/Chart.yaml +++ b/charts/s2-lite-helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: s2-lite-helm description: Self-hostable S2 streaming datastore using SlateDB on object storage type: application -version: 0.1.71 -appVersion: "0.43.0" +version: 0.1.64 +appVersion: "0.42.8" keywords: - s2 - streaming diff --git a/charts/s2-lite-helm/README.md b/charts/s2-lite-helm/README.md index ae02ba60..e019e02e 100644 --- a/charts/s2-lite-helm/README.md +++ b/charts/s2-lite-helm/README.md @@ -54,40 +54,6 @@ helm install my-s2-lite s2/s2-lite-helm \ Supports AWS S3, MinIO, Tigris, Cloudflare R2, and other S3-compatible services. -### Persistent volume - -```bash -helm install my-s2-lite s2/s2-lite-helm \ - --set persistentVolume.enabled=true \ - --set persistentVolume.size=20Gi -``` - -Stores the database on a `PersistentVolumeClaim` instead of a bucket. Set -`persistentVolume.existingClaim` to use your own claim. - -### Separate WAL storage - -The write-ahead log can live in a separate bucket or on its own persistent -volume (see [Storage](../../README.md#storage)). Requires `objectStorage` or -`persistentVolume`. - -```bash -# WAL on a persistent volume -helm install my-s2-lite s2/s2-lite-helm \ - --set objectStorage.enabled=true \ - --set objectStorage.bucket=my-bucket \ - --set walStorage.persistentVolume.enabled=true - -# WAL in a separate bucket -helm install my-s2-lite s2/s2-lite-helm \ - --set objectStorage.enabled=true \ - --set objectStorage.bucket=my-bucket \ - --set walStorage.bucket=my-wal-bucket -``` - -Keep the WAL location the same across upgrades; changing it does not migrate -WAL data and the server starts without an error. - ### TLS Configuration **Self-signed certificate (for dev/testing):** @@ -138,11 +104,6 @@ Common configurations: | `objectStorage.enabled` | Enable S3-compatible storage | `false` | | `objectStorage.bucket` | S3 bucket name | `""` | | `objectStorage.path` | Path prefix within bucket | `""` | -| `persistentVolume.enabled` | Store the database on a persistent volume | `false` | -| `persistentVolume.size` | Volume size | `10Gi` | -| `walStorage.bucket` | Separate S3 bucket for the WAL | `""` | -| `walStorage.persistentVolume.enabled` | Store the WAL on a persistent volume | `false` | -| `walStorage.persistentVolume.size` | WAL volume size | `10Gi` | | `metrics.serviceMonitor.enabled` | Enable Prometheus ServiceMonitor | `false` | ## Examples @@ -164,21 +125,6 @@ serviceAccount: helm install my-s2-lite s2/s2-lite-helm -f values.yaml ``` -### S3 with a local WAL volume - -```yaml -# values.yaml -objectStorage: - enabled: true - bucket: my-s3-bucket - -walStorage: - persistentVolume: - enabled: true - size: 20Gi - storageClass: gp3 -``` - ### Behind AWS Network Load Balancer ```yaml @@ -219,14 +165,3 @@ helm upgrade my-s2-lite s2/s2-lite-helm --version 0.1.0 helm uninstall my-s2-lite ``` -This deletes any `PersistentVolumeClaim` the chart created, and with it the data on the -volume. To keep the claim, annotate it before uninstalling: - -```yaml -persistentVolume: - annotations: - helm.sh/resource-policy: keep -``` - -Claims supplied via `existingClaim` are never deleted by the chart. - diff --git a/charts/s2-lite-helm/templates/NOTES.txt b/charts/s2-lite-helm/templates/NOTES.txt index ed9e7eb8..84b9755c 100644 --- a/charts/s2-lite-helm/templates/NOTES.txt +++ b/charts/s2-lite-helm/templates/NOTES.txt @@ -11,9 +11,7 @@ s2-lite is now running with the following configuration: - Replicas: {{ .Values.replicaCount }} - Image: {{ include "s2-lite.image" . }} -{{- $storageMode := include "s2-lite.storage.mode" . }} -{{- $walMode := include "s2-lite.walStorage.mode" . }} -{{- if eq $storageMode "bucket" }} +{{- if and .Values.objectStorage.enabled .Values.objectStorage.bucket }} - Storage: S3-compatible ({{ .Values.objectStorage.bucket }}) {{- if .Values.objectStorage.path }} - Path: {{ .Values.objectStorage.path }} @@ -21,16 +19,9 @@ s2-lite is now running with the following configuration: {{- if .Values.objectStorage.endpoint }} - Endpoint: {{ .Values.objectStorage.endpoint }} {{- end }} -{{- else if eq $storageMode "volume" }} - - Storage: Persistent volume ({{ include "s2-lite.claimName" (dict "root" . "name" "data" "volume" .Values.persistentVolume) }} at {{ .Values.persistentVolume.mountPath }}) {{- else }} - Storage: In-memory (data lost on pod restart) {{- end }} -{{- if eq $walMode "bucket" }} - - WAL: S3-compatible ({{ .Values.walStorage.bucket }}) -{{- else if eq $walMode "volume" }} - - WAL: Persistent volume ({{ include "s2-lite.claimName" (dict "root" . "name" "wal" "volume" .Values.walStorage.persistentVolume) }} at {{ .Values.walStorage.persistentVolume.mountPath }}) -{{- end }} Get the application URL by running these commands: export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "s2-lite.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") diff --git a/charts/s2-lite-helm/templates/_helpers.tpl b/charts/s2-lite-helm/templates/_helpers.tpl index faf28de6..62d8dd83 100644 --- a/charts/s2-lite-helm/templates/_helpers.tpl +++ b/charts/s2-lite-helm/templates/_helpers.tpl @@ -66,52 +66,3 @@ Create the image name {{- $tag := .Values.image.tag | default .Chart.AppVersion }} {{- printf "%s:%s" .Values.image.repository $tag }} {{- end }} - -{{/* -Main storage mode: "bucket", "volume", or "" for in-memory. -*/}} -{{- define "s2-lite.storage.mode" -}} -{{- if and .Values.objectStorage.enabled .Values.persistentVolume.enabled }} -{{- fail "objectStorage.enabled and persistentVolume.enabled are mutually exclusive" }} -{{- end }} -{{- if and .Values.objectStorage.enabled (not .Values.objectStorage.bucket) }} -{{- fail "objectStorage.enabled is true but objectStorage.bucket is not set" }} -{{- end }} -{{- if .Values.objectStorage.enabled }}bucket{{- else if .Values.persistentVolume.enabled }}volume{{- end }} -{{- end }} - -{{/* -WAL storage mode: "bucket", "volume", or "" when the WAL shares the main store. -*/}} -{{- define "s2-lite.walStorage.mode" -}} -{{- $bucket := .Values.walStorage.bucket }} -{{- $volume := .Values.walStorage.persistentVolume.enabled }} -{{- if and $bucket $volume }} -{{- fail "walStorage.bucket and walStorage.persistentVolume.enabled are mutually exclusive" }} -{{- end }} -{{- if and (or $bucket $volume) (not (include "s2-lite.storage.mode" .)) }} -{{- fail "walStorage requires objectStorage or persistentVolume" }} -{{- end }} -{{- if $bucket }}bucket{{- else if $volume }}volume{{- end }} -{{- end }} - -{{/* -Persistent volumes in use, keyed by volume name. Parse with fromYaml. -*/}} -{{- define "s2-lite.persistentVolumes" -}} -{{- $volumes := dict }} -{{- if eq (include "s2-lite.storage.mode" .) "volume" }} -{{- $_ := set $volumes "data" .Values.persistentVolume }} -{{- end }} -{{- if eq (include "s2-lite.walStorage.mode" .) "volume" }} -{{- $_ := set $volumes "wal" .Values.walStorage.persistentVolume }} -{{- end }} -{{- toYaml $volumes }} -{{- end }} - -{{/* -PersistentVolumeClaim name for a volume. Takes (dict "root" $ "name" "volume" ). -*/}} -{{- define "s2-lite.claimName" -}} -{{- .volume.existingClaim | default (printf "%s-%s" (include "s2-lite.fullname" .root) .name) }} -{{- end }} diff --git a/charts/s2-lite-helm/templates/deployment.yaml b/charts/s2-lite-helm/templates/deployment.yaml index 377dadaa..34bee45b 100644 --- a/charts/s2-lite-helm/templates/deployment.yaml +++ b/charts/s2-lite-helm/templates/deployment.yaml @@ -1,6 +1,3 @@ -{{- $storageMode := include "s2-lite.storage.mode" . }} -{{- $walMode := include "s2-lite.walStorage.mode" . }} -{{- $volumes := include "s2-lite.persistentVolumes" . | fromYaml }} apiVersion: apps/v1 kind: Deployment metadata: @@ -55,42 +52,27 @@ spec: {{- fail "tls.enabled is true but neither tls.cert/tls.key nor tls.selfSigned are configured" }} {{- end }} {{- end }} - {{- if eq $storageMode "bucket" }} + {{- if .Values.objectStorage.enabled }} + {{- if not .Values.objectStorage.bucket }} + {{- fail "objectStorage.enabled is true but objectStorage.bucket is not set" }} + {{- end }} - --bucket - {{ .Values.objectStorage.bucket | quote }} - {{- with .Values.objectStorage.path }} + {{- if .Values.objectStorage.path }} - --path - - {{ . | quote }} + - {{ .Values.objectStorage.path | quote }} {{- end }} - {{- else if eq $storageMode "volume" }} - - --local-root - - {{ .Values.persistentVolume.mountPath | quote }} - {{- end }} - {{- if eq $walMode "bucket" }} - - --wal-bucket - - {{ .Values.walStorage.bucket | quote }} - {{- else if eq $walMode "volume" }} - - --wal-local-root - - {{ .Values.walStorage.persistentVolume.mountPath | quote }} {{- end }} ports: - name: http containerPort: {{ .Values.service.targetPort }} protocol: TCP - {{- if or (and .Values.objectStorage.enabled .Values.objectStorage.endpoint) .Values.walStorage.endpoint .Values.walStorage.region .Values.env }} + {{- if or (and .Values.objectStorage.enabled .Values.objectStorage.endpoint) .Values.env }} env: {{- if and .Values.objectStorage.enabled .Values.objectStorage.endpoint }} - name: AWS_ENDPOINT_URL_S3 value: {{ .Values.objectStorage.endpoint | quote }} {{- end }} - {{- with .Values.walStorage.endpoint }} - - name: S2LITE_WAL_AWS_ENDPOINT_URL_S3 - value: {{ . | quote }} - {{- end }} - {{- with .Values.walStorage.region }} - - name: S2LITE_WAL_AWS_REGION - value: {{ . | quote }} - {{- end }} {{- with .Values.env }} {{- toYaml . | nindent 8 }} {{- end }} @@ -126,27 +108,14 @@ spec: {{- end }} resources: {{- toYaml .Values.resources | nindent 10 }} - {{- if or $volumes .Values.volumeMounts }} - volumeMounts: - {{- range $name, $volume := $volumes }} - - name: {{ $name }} - mountPath: {{ $volume.mountPath | quote }} - {{- end }} {{- with .Values.volumeMounts }} + volumeMounts: {{- toYaml . | nindent 8 }} {{- end }} - {{- end }} - {{- if or $volumes .Values.volumes }} - volumes: - {{- range $name, $volume := $volumes }} - - name: {{ $name }} - persistentVolumeClaim: - claimName: {{ include "s2-lite.claimName" (dict "root" $ "name" $name "volume" $volume) }} - {{- end }} {{- with .Values.volumes }} + volumes: {{- toYaml . | nindent 6 }} {{- end }} - {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/s2-lite-helm/templates/pvc.yaml b/charts/s2-lite-helm/templates/pvc.yaml deleted file mode 100644 index b19594c1..00000000 --- a/charts/s2-lite-helm/templates/pvc.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{{- range $name, $volume := include "s2-lite.persistentVolumes" . | fromYaml }} -{{- if not $volume.existingClaim }} ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "s2-lite.claimName" (dict "root" $ "name" $name "volume" $volume) }} - labels: - {{- include "s2-lite.labels" $ | nindent 4 }} - {{- with $volume.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - accessModes: - - ReadWriteOnce - {{- with $volume.storageClass }} - storageClassName: {{ . | quote }} - {{- end }} - resources: - requests: - storage: {{ $volume.size | quote }} -{{- end }} -{{- end }} diff --git a/charts/s2-lite-helm/values.yaml b/charts/s2-lite-helm/values.yaml index 22336d60..19087c99 100644 --- a/charts/s2-lite-helm/values.yaml +++ b/charts/s2-lite-helm/values.yaml @@ -102,8 +102,7 @@ ingress: # Storage Configuration # # By default, s2-lite runs in-memory (great for dev/testing). -# For persistent data, enable objectStorage with an S3-compatible bucket or -# persistentVolume for a local disk. Enable at most one. +# For persistent data, enable objectStorage with an S3-compatible bucket. # ------------------------------------------------------------------------------ objectStorage: @@ -119,45 +118,6 @@ objectStorage: # R2: "https://.r2.cloudflarestorage.com" endpoint: "" -# Persistent volume for the database (--local-root) -persistentVolume: - enabled: false - mountPath: /data - size: 10Gi - storageClass: "" - # Use an existing PersistentVolumeClaim instead of creating one - existingClaim: "" - # helm uninstall deletes the created claim; add helm.sh/resource-policy: keep to retain it - annotations: {} - -# ------------------------------------------------------------------------------ -# WAL Storage Configuration -# -# By default the write-ahead log (WAL) is stored with the database. Place it in -# a separate bucket or on its own persistent volume to isolate WAL latency from -# flushes and compaction. Requires objectStorage or persistentVolume; set at -# most one of bucket or persistentVolume.enabled. -# -# Keep the WAL location the same across upgrades: changing it does not migrate -# WAL data and the server starts without an error. -# ------------------------------------------------------------------------------ - -walStorage: - # S3 bucket for the WAL (--wal-bucket). Shares objectStorage's AWS - # configuration unless overridden below. For different credentials, set - # S2LITE_WAL_AWS_ACCESS_KEY_ID / S2LITE_WAL_AWS_SECRET_ACCESS_KEY via env. - bucket: "" - endpoint: "" - region: "" - # Persistent volume for the WAL (--wal-local-root) - persistentVolume: - enabled: false - mountPath: /wal - size: 10Gi - storageClass: "" - existingClaim: "" - annotations: {} - # ------------------------------------------------------------------------------ # Observability # ------------------------------------------------------------------------------ diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 9004262c..9dfeac91 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -2,54 +2,6 @@ All notable changes to this project will be documented in this file. -## [0.43.0] - 2026-09-25 - -### Features - -- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) - -### Bug Fixes - -- Show match-none token scopes as no access, not as wildcards ([#782](https://github.com/s2-streamstore/s2/issues/782)) - - - -## [0.42.14] - 2026-09-24 - - - -## [0.42.13] - 2026-09-22 - -### Features - -- Support a separate WAL object store ([#766](https://github.com/s2-streamstore/s2/issues/766)) - - - -## [0.42.12] - 2026-09-16 - -### Miscellaneous Tasks - -- Update Cargo.lock dependencies - - - -## [0.42.11] - 2026-09-11 - -### Features - -- `s2-stream-config` header for auto-created streams ([#718](https://github.com/s2-streamstore/s2/issues/718)) - - - -## [0.42.10] - 2026-09-11 - - - -## [0.42.9] - 2026-09-10 - - - ## [0.42.8] - 2026-09-01 ### Bug Fixes diff --git a/cli/Cargo.toml b/cli/Cargo.toml index b5284ce4..64dfddeb 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-cli" -version = "0.43.0" +version = "0.42.8" description = "CLI for S2" edition.workspace = true license.workspace = true @@ -21,7 +21,6 @@ bytes = { workspace = true } clap = { workspace = true, features = ["derive"] } color-print = { workspace = true } colored = { workspace = true } -compact_str = { workspace = true, features = ["serde"] } config = { workspace = true } dirs = { workspace = true } fs2 = { workspace = true } diff --git a/cli/schema.json b/cli/schema.json index c785bd21..c6f9c997 100644 --- a/cli/schema.json +++ b/cli/schema.json @@ -87,10 +87,14 @@ "type": "object", "properties": { "storage_class": { - "description": "[Storage class](https://s2.dev/docs/storage-classes) for recent writes.", - "type": [ - "string", - "null" + "description": "Storage class for recent writes.", + "anyOf": [ + { + "$ref": "#/$defs/StorageClass" + }, + { + "type": "null" + } ], "default": null }, @@ -130,6 +134,14 @@ }, "additionalProperties": false }, + "StorageClass": { + "description": "Storage class for recent writes.", + "type": "string", + "enum": [ + "standard", + "express" + ] + }, "RetentionPolicy": { "description": "Retain records unless explicitly trimmed (\"infinite\"), or automatically trim records older than the given duration (e.g. \"7days\", \"1week\"). Age durations must be greater than 0 seconds.", "type": "string", diff --git a/cli/src/apply.rs b/cli/src/apply.rs index 372b2d0e..7235508d 100644 --- a/cli/src/apply.rs +++ b/cli/src/apply.rs @@ -3,19 +3,88 @@ use std::path::Path; use colored::Colorize; -use miette::IntoDiagnostic; use s2_common::{ basin::BasinName, - config::{RetentionPolicy, StreamConfig, TimestampingMode}, + config::{ + BasinConfig, OptionalDeleteOnEmptyConfig, OptionalStreamConfig, OptionalTimestampingConfig, + RetentionPolicy, StorageClass, StreamConfig, TimestampingMode, + }, encryption::EncryptionAlgorithm, stream::StreamName, }; -use s2_sdk::{ - error::{ErrorCode, RequestError, StatusCode}, - types::BasinConfig, -}; +use s2_sdk::error::{ErrorCode, RequestError}; -use crate::types::resolve_stream_config; +fn basin_config_from_sdk(config: s2_sdk::types::BasinConfig) -> BasinConfig { + BasinConfig { + default_stream_config: config + .default_stream_config + .map(optional_stream_config_from_sdk) + .unwrap_or_default(), + stream_cipher: config.stream_cipher, + create_stream_on_append: config.create_stream_on_append, + create_stream_on_read: config.create_stream_on_read, + } +} + +fn stream_config_from_sdk(config: s2_sdk::types::StreamConfig) -> StreamConfig { + optional_stream_config_from_sdk(config).into() +} + +fn optional_stream_config_from_sdk(config: s2_sdk::types::StreamConfig) -> OptionalStreamConfig { + OptionalStreamConfig { + storage_class: config.storage_class.map(storage_class_from_sdk), + retention_policy: config.retention_policy.map(retention_policy_from_sdk), + timestamping: config + .timestamping + .map(timestamping_from_sdk) + .unwrap_or_default(), + delete_on_empty: config + .delete_on_empty + .map(delete_on_empty_from_sdk) + .unwrap_or_default(), + } +} + +fn storage_class_from_sdk(storage_class: s2_sdk::types::StorageClass) -> StorageClass { + match storage_class { + s2_sdk::types::StorageClass::Standard => StorageClass::Standard, + s2_sdk::types::StorageClass::Express => StorageClass::Express, + } +} + +fn retention_policy_from_sdk(retention_policy: s2_sdk::types::RetentionPolicy) -> RetentionPolicy { + match retention_policy { + s2_sdk::types::RetentionPolicy::Age(secs) => { + RetentionPolicy::Age(std::time::Duration::from_secs(secs)) + } + s2_sdk::types::RetentionPolicy::Infinite => RetentionPolicy::Infinite(), + } +} + +fn timestamping_from_sdk( + timestamping: s2_sdk::types::TimestampingConfig, +) -> OptionalTimestampingConfig { + OptionalTimestampingConfig { + mode: timestamping.mode.map(timestamping_mode_from_sdk), + uncapped: timestamping.uncapped, + } +} + +fn timestamping_mode_from_sdk(mode: s2_sdk::types::TimestampingMode) -> TimestampingMode { + match mode { + s2_sdk::types::TimestampingMode::ClientPrefer => TimestampingMode::ClientPrefer, + s2_sdk::types::TimestampingMode::ClientRequire => TimestampingMode::ClientRequire, + s2_sdk::types::TimestampingMode::Arrival => TimestampingMode::Arrival, + } +} + +fn delete_on_empty_from_sdk( + delete_on_empty: s2_sdk::types::DeleteOnEmptyConfig, +) -> OptionalDeleteOnEmptyConfig { + OptionalDeleteOnEmptyConfig { + min_age: Some(std::time::Duration::from_secs(delete_on_empty.min_age_secs)), + } +} fn basin_config_to_sdk(config: s2_resource_spec::BasinConfig) -> s2_sdk::types::BasinConfig { let mut sdk_config = s2_sdk::types::BasinConfig::new(); @@ -38,7 +107,7 @@ fn basin_config_to_sdk(config: s2_resource_spec::BasinConfig) -> s2_sdk::types:: fn stream_config_to_sdk(config: s2_resource_spec::StreamConfig) -> s2_sdk::types::StreamConfig { let mut sdk_config = s2_sdk::types::StreamConfig::new(); if let Some(storage_class) = config.storage_class { - sdk_config = sdk_config.with_storage_class(storage_class); + sdk_config = sdk_config.with_storage_class(storage_class_to_sdk(storage_class)); } if let Some(retention_policy) = config.retention_policy { sdk_config = sdk_config.with_retention_policy(retention_policy_to_sdk(retention_policy)); @@ -52,6 +121,15 @@ fn stream_config_to_sdk(config: s2_resource_spec::StreamConfig) -> s2_sdk::types sdk_config } +fn storage_class_to_sdk( + storage_class: s2_resource_spec::StorageClass, +) -> s2_sdk::types::StorageClass { + match storage_class { + s2_resource_spec::StorageClass::Standard => s2_sdk::types::StorageClass::Standard, + s2_resource_spec::StorageClass::Express => s2_sdk::types::StorageClass::Express, + } +} + fn retention_policy_to_sdk( retention_policy: s2_resource_spec::RetentionPolicy, ) -> s2_sdk::types::RetentionPolicy { @@ -212,26 +290,10 @@ async fn apply_stream( enum ResourceAction { Create, - Ensure { - diffs: Vec, - storage_class_unresolved: bool, - }, + Ensure(Vec), Unchanged, } -impl ResourceAction { - fn from_diff(diffs: Vec, storage_class_unresolved: bool) -> Self { - if diffs.is_empty() && !storage_class_unresolved { - Self::Unchanged - } else { - Self::Ensure { - diffs, - storage_class_unresolved, - } - } - } -} - struct FieldDiff { field: &'static str, old: String, @@ -245,6 +307,13 @@ fn is_not_found_error(error: &RequestError) -> bool { .is_some_and(|code| matches!(code, ErrorCode::BasinNotFound | ErrorCode::StreamNotFound)) } +fn format_storage_class(sc: StorageClass) -> &'static str { + match sc { + StorageClass::Standard => "standard", + StorageClass::Express => "express", + } +} + fn format_retention_policy(rp: RetentionPolicy) -> String { match rp { RetentionPolicy::Age(age) => humantime::format_duration(age).to_string(), @@ -271,10 +340,7 @@ fn default_stream_config_field(field: &'static str) -> &'static str { } } -fn diff_basin_config( - existing: &BasinConfig, - desired: &BasinConfig, -) -> miette::Result> { +fn diff_basin_config(existing: &BasinConfig, desired: &BasinConfig) -> Vec { let mut diffs = Vec::new(); if existing.stream_cipher != desired.stream_cipher { @@ -309,24 +375,8 @@ fn diff_basin_config( }); } - let existing_dsc = resolve_stream_config( - existing - .default_stream_config - .clone() - .unwrap_or_default() - .into(), - Default::default(), - ) - .into_diagnostic()?; - let desired_dsc = resolve_stream_config( - desired - .default_stream_config - .clone() - .unwrap_or_default() - .into(), - Default::default(), - ) - .into_diagnostic()?; + let existing_dsc: StreamConfig = existing.default_stream_config.clone().into(); + let desired_dsc: StreamConfig = desired.default_stream_config.clone().into(); for sd in diff_stream_configs(&existing_dsc, &desired_dsc) { diffs.push(FieldDiff { field: default_stream_config_field(sd.field), @@ -335,23 +385,17 @@ fn diff_basin_config( }); } - Ok(diffs) + diffs } fn diff_stream_configs(existing: &StreamConfig, desired: &StreamConfig) -> Vec { let mut diffs = Vec::new(); - if let Some(storage_class) = &desired.storage_class - && existing.storage_class.as_ref() != Some(storage_class) - { + if existing.storage_class != desired.storage_class { diffs.push(FieldDiff { field: "storage_class", - old: existing - .storage_class - .as_deref() - .unwrap_or("unspecified") - .to_owned(), - new: storage_class.to_string(), + old: format_storage_class(existing.storage_class).to_string(), + new: format_storage_class(desired.storage_class).to_string(), }); } @@ -434,7 +478,7 @@ fn spec_stream_fields(spec: &s2_resource_spec::StreamConfig) -> Vec { fields.push(FieldDiff { field: "storage_class", old: String::new(), - new: sc.to_string(), + new: format_storage_class(sc.clone().into()).to_string(), }); } if let Some(ref rp) = spec.retention_policy { @@ -478,18 +522,11 @@ fn print_basin_result(basin: &str, action: &ResourceAction) { ResourceAction::Create => { println!("{}", format!("+ basin {basin}").green().bold()); } - ResourceAction::Ensure { - diffs, - storage_class_unresolved, - } => { - let marker = if diffs.is_empty() { "?" } else { "~" }; - println!("{}", format!("{marker} basin {basin}").yellow().bold()); + ResourceAction::Ensure(diffs) => { + println!("{}", format!("~ basin {basin}").yellow().bold()); for diff in diffs { println!(" {}: {} → {}", diff.field, diff.old.dimmed(), diff.new); } - if *storage_class_unresolved { - println!(" default_stream_config.storage_class: resolved at apply"); - } } ResourceAction::Unchanged => { println!("{}", format!("= basin {basin}").dimmed()); @@ -502,23 +539,11 @@ fn print_stream_result(basin: &str, stream: &str, action: &ResourceAction) { ResourceAction::Create => { println!("{}", format!(" + stream {basin}/{stream}").green().bold()); } - ResourceAction::Ensure { - diffs, - storage_class_unresolved, - } => { - let marker = if diffs.is_empty() { "?" } else { "~" }; - println!( - "{}", - format!(" {marker} stream {basin}/{stream}") - .yellow() - .bold() - ); + ResourceAction::Ensure(diffs) => { + println!("{}", format!(" ~ stream {basin}/{stream}").yellow().bold()); for diff in diffs { println!(" {}: {} → {}", diff.field, diff.old.dimmed(), diff.new); } - if *storage_class_unresolved { - println!(" storage_class: resolved at apply"); - } } ResourceAction::Unchanged => { println!("{}", format!(" = stream {basin}/{stream}").dimmed()); @@ -547,55 +572,24 @@ fn print_stream_create(basin: &str, stream: &str, spec: &Option miette::Result<()> { validate(&spec)?; - let needs_location_default = spec.basins.iter().any(|basin| { - basin - .config - .as_ref() - .and_then(|config| config.default_stream_config.as_ref()) - .and_then(|config| config.storage_class.as_ref()) - .is_none() - }); - let default_storage_class = if needs_location_default { - match s2.get_default_location().await { - Ok(location) => location.default_storage_class, - Err(error) - if error.server_error().is_some_and(|error| { - matches!( - error.known_code(), - Some(ErrorCode::NotImplemented | ErrorCode::PermissionDenied) - ) || error.status == StatusCode::NOT_FOUND - }) => - { - None - } - Err(error) => { - return Err(miette::miette!( - "failed to get default storage class: {error}" - )); - } - } - } else { - None - }; - for basin_spec in spec.basins { - let mut desired_basin_config = basin_spec + let desired_basin_config = basin_spec .config .clone() - .map(basin_config_to_sdk) + .map(BasinConfig::from) .unwrap_or_default(); - let desired_basin_defaults = desired_basin_config - .default_stream_config - .get_or_insert_default(); - if desired_basin_defaults.storage_class.is_none() { - desired_basin_defaults.storage_class = default_storage_class.clone(); - } - let desired_basin_defaults = desired_basin_defaults.clone(); + let desired_basin_default_stream_config = + desired_basin_config.default_stream_config.clone(); let basin_action = match s2.get_basin_config(basin_spec.name.clone()).await { Ok(existing) => { - let diffs = diff_basin_config(&existing, &desired_basin_config)?; - ResourceAction::from_diff(diffs, desired_basin_defaults.storage_class.is_none()) + let existing = basin_config_from_sdk(existing); + let diffs = diff_basin_config(&existing, &desired_basin_config); + if diffs.is_empty() { + ResourceAction::Unchanged + } else { + ResourceAction::Ensure(diffs) + } } Err(e) if is_not_found_error(&e) => ResourceAction::Create, Err(e) => { @@ -624,20 +618,19 @@ pub async fn dry_run(s2: &s2_sdk::S2, spec: s2_resource_spec::Resources) -> miet .await { Ok(existing) => { - let existing = resolve_stream_config(existing.into(), Default::default()) - .into_diagnostic()?; - let desired_stream_config = resolve_stream_config( - stream_spec - .config - .clone() - .map(stream_config_to_sdk) - .unwrap_or_default() - .into(), - desired_basin_defaults.clone().into(), - ) - .into_diagnostic()?; + let existing = stream_config_from_sdk(existing); + let desired_stream_config = stream_spec + .config + .clone() + .map(OptionalStreamConfig::from) + .unwrap_or_default() + .merge(desired_basin_default_stream_config.clone()); let diffs = diff_stream_configs(&existing, &desired_stream_config); - ResourceAction::from_diff(diffs, desired_stream_config.storage_class.is_none()) + if diffs.is_empty() { + ResourceAction::Unchanged + } else { + ResourceAction::Ensure(diffs) + } } Err(e) if is_not_found_error(&e) => ResourceAction::Create, Err(e) => { diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 30f2baf7..74981c1a 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -1,7 +1,6 @@ use std::{num::NonZeroU64, path::PathBuf, time::Duration}; use clap::{Args, Parser, Subcommand, ValueEnum, builder::styling}; -use compact_str::CompactString; use s2_sdk::types::{ AccessTokenId, AccessTokenIdPrefix, AccessTokenIdStartAfter, BasinName, BasinNamePrefix, BasinNameStartAfter, EncryptionAlgorithm, EncryptionKey, FencingToken, StreamName, @@ -15,7 +14,7 @@ use crate::{ }, types::{ BasinConfig, Interval, LocationName, Operation, PermittedOperationGroups, - S2BasinAndMaybeStreamUri, S2BasinAndStreamUri, S2BasinUri, StreamConfig, + S2BasinAndMaybeStreamUri, S2BasinAndStreamUri, S2BasinUri, StorageClass, StreamConfig, }, }; @@ -192,7 +191,6 @@ pub enum Command { /// `+` create /// `~` ensure /// `=` unchanged - /// `?` storage-class default resolved at apply /// /// For IDE validation/autocomplete, add `$schema` at the top of each spec file: /// {"$schema":"https://raw.githubusercontent.com/s2-streamstore/s2/main/cli/schema.json","basins":[]} @@ -695,15 +693,6 @@ pub struct AppendArgs { #[command(flatten)] pub encryption_key: EncryptionKeyArgs, - - /// Stream configuration to apply if the stream is created on append. - /// Unset fields inherit the basin's default stream configuration. - /// Ignored if the stream already exists. - #[command( - flatten, - next_help_heading = "Stream configuration (applied only if the stream is created on append)" - )] - pub stream_config: StreamConfig, } #[derive(Args, Debug, Clone, Default)] @@ -782,15 +771,6 @@ pub struct ReadArgs { #[command(flatten)] pub encryption_key: EncryptionKeyArgs, - - /// Stream configuration to apply if the stream is created on read. - /// Unset fields inherit the basin's default stream configuration. - /// Ignored if the stream already exists. - #[command( - flatten, - next_help_heading = "Stream configuration (applied only if the stream is created on read)" - )] - pub stream_config: StreamConfig, } #[derive(Args, Debug)] @@ -836,7 +816,6 @@ pub struct ApplyArgs { /// `+` create /// `~` ensure /// `=` unchanged - /// `?` storage-class default resolved at apply #[arg(long)] pub dry_run: bool, /// Print the JSON Schema for the spec file format to stdout. @@ -851,7 +830,7 @@ pub struct BenchArgs { /// Storage class for the test stream. Uses basin default if not specified. #[arg(short = 'c', long)] - pub storage_class: Option, + pub storage_class: Option, /// Total metered record size in bytes (includes headers and overhead). #[arg( diff --git a/cli/src/diff.rs b/cli/src/diff.rs index 6ea1ffe3..75f612a4 100644 --- a/cli/src/diff.rs +++ b/cli/src/diff.rs @@ -1,12 +1,16 @@ -//! Diffing over typed presentation views of basins, streams, and access tokens. +//! Diffing over typed presentation views of Basins, Streams, and Access Tokens. +//! +//! - Configs are fetched as API DTOs and converted into `s2_common` domain types via the existing +//! `TryFrom` conversions, which also resolve omitted fields to their effective defaults (e.g. an +//! unset `storage_class` becomes `express`). +//! +//! - Domain values are rendered into `*View` structs and serialized to JSON, which is diffed +//! field-by-field. Views own all presentation: compact exact durations (`"7d"`, `"1h"`), +//! wire-format enum names, canonically ordered operation sets. use colored::Colorize; -use compact_str::CompactString; use s2_api::v1::access::AccessTokenInfo as ApiAccessTokenInfo; -use s2_common::{ - access::{PermittedOperationGroups, ReadWritePermissions, ResourceSet}, - config::StreamConfig, -}; +use s2_common::access::{PermittedOperationGroups, ReadWritePermissions, ResourceSet}; use s2_sdk::types::AccessTokenId; use serde::Serialize; use serde_json::{Map, Value}; @@ -15,7 +19,7 @@ use crate::{ cli::{DiffArgs, DiffOutput, DiffResourceKind}, error::CliError, ops, - types::{DiffResource, S2BasinAndStreamUri, S2BasinUri, resolve_stream_config}, + types::{DiffResource, S2BasinAndStreamUri, S2BasinUri}, }; #[derive(Debug, Serialize)] @@ -294,7 +298,7 @@ fn resource_kind(resource: &DiffResource) -> DiffResourceKind { } } -/// Presentation of a stream configuration for comparison. +/// Presentation of an effective (fully resolved) stream configuration. /// /// - Field names and nesting mirror the API wire format so that `--output json` paths line up with /// API responses. @@ -302,7 +306,7 @@ fn resource_kind(resource: &DiffResource) -> DiffResourceKind { /// duration. #[derive(Serialize)] struct StreamConfigView { - storage_class: Option, + storage_class: String, retention_policy: String, timestamping: TimestampingConfigView, delete_on_empty: DeleteOnEmptyConfigView, @@ -319,9 +323,9 @@ struct DeleteOnEmptyConfigView { min_age: String, } -impl From for StreamConfigView { - fn from(config: StreamConfig) -> Self { - let StreamConfig { +impl From for StreamConfigView { + fn from(config: s2_common::config::StreamConfig) -> Self { + let s2_common::config::StreamConfig { storage_class, retention_policy, timestamping, @@ -330,7 +334,7 @@ impl From for StreamConfigView { let s2_common::config::TimestampingConfig { mode, uncapped } = timestamping; Self { - storage_class, + storage_class: wire_name(s2_api::v1::config::StorageClass::from(storage_class)), retention_policy: match retention_policy { s2_common::config::RetentionPolicy::Age(age) => compact_duration(age), s2_common::config::RetentionPolicy::Infinite() => "infinite".to_owned(), @@ -355,30 +359,25 @@ struct BasinConfigView { create_stream_on_read: bool, } -impl TryFrom for BasinConfigView { - type Error = s2_common::ValidationError; - - fn try_from(config: s2_api::v1::config::BasinConfig) -> Result { - let s2_api::v1::config::BasinConfig { +impl From for BasinConfigView { + fn from(config: s2_common::config::BasinConfig) -> Self { + let s2_common::config::BasinConfig { default_stream_config, stream_cipher, create_stream_on_append, create_stream_on_read, } = config; - Ok(Self { - default_stream_config: resolve_stream_config( - default_stream_config.unwrap_or_default(), - Default::default(), - )? - .into(), + Self { + default_stream_config: s2_common::config::StreamConfig::from(default_stream_config) + .into(), stream_cipher: match stream_cipher { None => "none".to_owned(), - Some(cipher) => wire_name(cipher), + Some(cipher) => wire_name(s2_api::v1::config::EncryptionAlgorithm::from(cipher)), }, create_stream_on_append, create_stream_on_read, - }) + } } } @@ -465,16 +464,20 @@ fn permissions_view(permissions: ReadWritePermissions) -> String { } fn basin_view(config: s2_api::v1::config::BasinConfig) -> Result { - Ok(serde_json::to_value(BasinConfigView::try_from(config)?)?) + let config: s2_common::config::BasinConfig = config.try_into()?; + Ok(serde_json::to_value(BasinConfigView::from(config))?) } /// Renders a basin's effective stream defaults for comparison against a concrete stream. fn basin_stream_defaults_view(config: s2_api::v1::config::BasinConfig) -> Result { - stream_view(config.default_stream_config.unwrap_or_default()) + let config: s2_common::config::BasinConfig = config.try_into()?; + let defaults = s2_common::config::StreamConfig::from(config.default_stream_config); + Ok(serde_json::to_value(StreamConfigView::from(defaults))?) } fn stream_view(config: s2_api::v1::config::StreamConfig) -> Result { - let config = resolve_stream_config(config, Default::default())?; + let config: s2_common::config::OptionalStreamConfig = config.try_into()?; + let config = s2_common::config::StreamConfig::from(config); Ok(serde_json::to_value(StreamConfigView::from(config))?) } @@ -738,7 +741,7 @@ mod tests { assert_eq!( view, json!({ - "storage_class": null, + "storage_class": "express", "retention_policy": "2d", "timestamping": {"mode": "client-prefer", "uncapped": false}, "delete_on_empty": {"min_age": "1h"} @@ -770,7 +773,7 @@ mod tests { fn omitted_and_explicit_defaults_render_identically() { let omitted = serde_json::from_value(json!({})).expect("empty config deserializes"); let explicit = serde_json::from_value(json!({ - "storage_class": null, + "storage_class": "express", "retention_policy": {"age": 604800}, "timestamping": {"mode": "client-prefer", "uncapped": false}, "delete_on_empty": {"min_age_secs": 0} @@ -799,7 +802,7 @@ mod tests { view, json!({ "default_stream_config": { - "storage_class": null, + "storage_class": "express", "retention_policy": "7d", "timestamping": {"mode": "client-prefer", "uncapped": false}, "delete_on_empty": {"min_age": "0s"} diff --git a/cli/src/main.rs b/cli/src/main.rs index 7c23f771..cc809202 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -42,7 +42,7 @@ use s2_sdk::{ S2, types::{ AppendRetryPolicy, CreateStreamInput, DeleteOnEmptyConfig, DeleteStreamInput, - EncryptionKey, LocationInfo, MeteredBytes, Metric, RetentionPolicy, RetryConfig, + EncryptionKey, MeteredBytes, Metric, RetentionPolicy, RetryConfig, StreamConfig as SdkStreamConfig, StreamName, TimestampingConfig, TimestampingMode, }, }; @@ -531,13 +531,13 @@ async fn run(cli: Cli) -> Result { Command::ListLocations => { let locations = ops::list_locations(&s2).await?; for location_info in locations { - print_location_listing(&location_info); + print_location_listing(location_info.name.to_string(), location_info.is_private); } } Command::GetDefaultLocation => { let location = ops::get_default_location(&s2).await?; - print_location_listing(&location); + print_location_listing(location.name.to_string(), location.is_private); } Command::SetDefaultLocation { location } => { @@ -549,7 +549,7 @@ async fn run(cli: Cli) -> Result { .green() .bold() ); - print_location_listing(&location); + print_location_listing(location.name.to_string(), location.is_private); } Command::GetAccountMetrics(args) => { @@ -657,13 +657,9 @@ async fn run(cli: Cli) -> Result { record_stream, args.uri, encryption_key.as_ref(), - ops::AppendOptions { - fencing_token: args.fencing_token, - match_seq_num: args.match_seq_num, - linger: *args.linger, - stream_config: (!args.stream_config.is_empty()) - .then(|| args.stream_config.into()), - }, + args.fencing_token, + args.match_seq_num, + *args.linger, ); let mut acks = std::pin::pin!(acks); let mut last_printed_batch_end: Option = None; @@ -852,7 +848,7 @@ async fn run(cli: Cli) -> Result { .with_mode(TimestampingMode::ClientRequire) .with_uncapped(true), ); - stream_config.storage_class = args.storage_class; + stream_config.storage_class = args.storage_class.map(Into::into); let s2 = S2::new(sdk_config.clone().with_retry( RetryConfig::new().with_append_retry_policy(AppendRetryPolicy::NoSideEffects), @@ -921,20 +917,9 @@ fn print_basin_listing(name: String, location: Option<&str>, is_deleting: bool) } } -fn print_location_listing(location: &LocationInfo) { - let visibility = format_location_visibility(location.is_private); - println!("{} {visibility}", location.name); - if let Some(storage_classes) = &location.storage_classes { - let classes = if storage_classes.is_empty() { - "none".to_owned() - } else { - storage_classes.join(", ") - }; - println!(" storage classes: {classes}"); - } - if let Some(default_storage_class) = &location.default_storage_class { - println!(" default storage class: {default_storage_class}"); - } +fn print_location_listing(name: String, is_private: bool) { + let visibility = format_location_visibility(is_private); + println!("{name} {visibility}"); } fn format_location_visibility(is_private: bool) -> colored::ColoredString { diff --git a/cli/src/ops.rs b/cli/src/ops.rs index 468d6ab0..7ae3218d 100644 --- a/cli/src/ops.rs +++ b/cli/src/ops.rs @@ -570,32 +570,23 @@ pub async fn read( stop = stop.with_until(..until); } - let mut input = ReadInput::new().with_start(start).with_stop(stop); - if !args.stream_config.is_empty() { - input = input.with_stream_config(args.stream_config.clone().into()); - } - stream - .read_session(input, ReadSessionConfig::default()) + .read_session( + ReadInput::new().with_start(start).with_stop(stop), + ReadSessionConfig::default(), + ) .await .map_err(|e| CliError::op(OpKind::Read, e)) } -/// Options controlling how records are appended. -pub struct AppendOptions { - pub fencing_token: Option, - pub match_seq_num: Option, - pub linger: Duration, - /// Stream configuration to apply if the stream is created on append. - pub stream_config: Option, -} - pub fn append<'a, S, E>( s2: &'a S2, records: S, uri: S2BasinAndStreamUri, encryption_key: Option<&'a EncryptionKey>, - options: AppendOptions, + fencing_token: Option, + match_seq_num: Option, + linger: Duration, ) -> impl Stream> + Send + 'a where S: Stream> + Send + Unpin + 'a, @@ -603,15 +594,12 @@ where { let stream = stream_with_encryption(s2, uri, encryption_key); - let batching_config = BatchingConfig::new().with_linger(options.linger); + let batching_config = BatchingConfig::new().with_linger(linger); let mut producer_config = ProducerConfig::new().with_batching(batching_config); - if let Some(config) = options.stream_config { - producer_config = producer_config.with_stream_config(config); - } - if let Some(ft) = options.fencing_token { + if let Some(ft) = fencing_token { producer_config = producer_config.with_fencing_token(ft); } - if let Some(seq) = options.match_seq_num { + if let Some(seq) = match_seq_num { producer_config = producer_config.with_match_seq_num(seq); } diff --git a/cli/src/types.rs b/cli/src/types.rs index d6187cbc..699a04b9 100644 --- a/cli/src/types.rs +++ b/cli/src/types.rs @@ -2,7 +2,6 @@ use std::{str::FromStr, time::Duration}; use clap::{Args, Parser, ValueEnum}; use colored::Colorize; -use compact_str::CompactString; use s2_sdk::{ self as sdk, types::{ @@ -175,7 +174,7 @@ pub struct BasinConfig { pub struct StreamConfig { #[arg(long)] /// Storage class for a stream. - pub storage_class: Option, + pub storage_class: Option, #[arg(long, help("Example: 1d, 1w, 1y"))] /// Retention policy for a stream. pub retention_policy: Option, @@ -202,16 +201,15 @@ impl StreamConfig { } } -pub fn resolve_stream_config( - config: s2_api::v1::config::StreamConfig, - basin_defaults: s2_api::v1::config::StreamConfig, -) -> Result { - let config: s2_common::config::OptionalStreamConfig = config.try_into()?; - Ok(config.merge(basin_defaults.try_into()?)) -} - pub use sdk::types::LocationName; +#[derive(ValueEnum, Debug, Clone, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum StorageClass { + Standard, + Express, +} + #[derive(ValueEnum, Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub enum TimestampingMode { @@ -304,7 +302,7 @@ impl From for sdk::types::StreamConfig { fn from(config: StreamConfig) -> Self { let mut stream_config = sdk::types::StreamConfig::new(); if let Some(storage_class) = config.storage_class { - stream_config = stream_config.with_storage_class(storage_class); + stream_config = stream_config.with_storage_class(storage_class.into()); } if let Some(retention_policy) = config.retention_policy { stream_config = stream_config.with_retention_policy(retention_policy.into()); @@ -319,6 +317,24 @@ impl From for sdk::types::StreamConfig { } } +impl From for sdk::types::StorageClass { + fn from(class: StorageClass) -> Self { + match class { + StorageClass::Standard => sdk::types::StorageClass::Standard, + StorageClass::Express => sdk::types::StorageClass::Express, + } + } +} + +impl From for StorageClass { + fn from(class: sdk::types::StorageClass) -> Self { + match class { + sdk::types::StorageClass::Standard => StorageClass::Standard, + sdk::types::StorageClass::Express => StorageClass::Express, + } + } +} + impl From for sdk::types::TimestampingMode { fn from(mode: TimestampingMode) -> Self { match mode { @@ -398,7 +414,7 @@ impl From for BasinConfig { impl From for StreamConfig { fn from(config: sdk::types::StreamConfig) -> Self { StreamConfig { - storage_class: config.storage_class, + storage_class: config.storage_class.map(Into::into), retention_policy: config.retention_policy.map(Into::into), timestamping: config.timestamping.map(Into::into), delete_on_empty: config.delete_on_empty.map(Into::into), @@ -410,7 +426,7 @@ impl From for sdk::types::StreamReconfiguration { fn from(config: StreamConfig) -> Self { let mut reconfig = sdk::types::StreamReconfiguration::new(); if let Some(storage_class) = config.storage_class { - reconfig = reconfig.with_storage_class(storage_class); + reconfig = reconfig.with_storage_class(storage_class.into()); } if let Some(retention_policy) = config.retention_policy { reconfig = reconfig.with_retention_policy(retention_policy.into()); @@ -481,35 +497,32 @@ where serializer.serialize_str(&humantime::format_duration(*value).to_string()) } -impl BasinMatcher { - /// Converts an SDK matcher, returning `None` for a matcher that matches no resources. - fn from_sdk(matcher: sdk::types::BasinMatcher) -> Option { +impl From for BasinMatcher { + fn from(matcher: sdk::types::BasinMatcher) -> Self { match matcher { - sdk::types::BasinMatcher::Exact(v) => Some(BasinMatcher::Exact(v)), - sdk::types::BasinMatcher::Prefix(v) => Some(BasinMatcher::Prefix(v)), - sdk::types::BasinMatcher::None => None, + sdk::types::BasinMatcher::Exact(v) => BasinMatcher::Exact(v), + sdk::types::BasinMatcher::Prefix(v) => BasinMatcher::Prefix(v), + sdk::types::BasinMatcher::None => BasinMatcher::Prefix(Default::default()), } } } -impl StreamMatcher { - /// Converts an SDK matcher, returning `None` for a matcher that matches no resources. - fn from_sdk(matcher: sdk::types::StreamMatcher) -> Option { +impl From for StreamMatcher { + fn from(matcher: sdk::types::StreamMatcher) -> Self { match matcher { - sdk::types::StreamMatcher::Exact(v) => Some(StreamMatcher::Exact(v)), - sdk::types::StreamMatcher::Prefix(v) => Some(StreamMatcher::Prefix(v)), - sdk::types::StreamMatcher::None => None, + sdk::types::StreamMatcher::Exact(v) => StreamMatcher::Exact(v), + sdk::types::StreamMatcher::Prefix(v) => StreamMatcher::Prefix(v), + sdk::types::StreamMatcher::None => StreamMatcher::Prefix(Default::default()), } } } -impl AccessTokenMatcher { - /// Converts an SDK matcher, returning `None` for a matcher that matches no resources. - fn from_sdk(matcher: sdk::types::AccessTokenMatcher) -> Option { +impl From for AccessTokenMatcher { + fn from(matcher: sdk::types::AccessTokenMatcher) -> Self { match matcher { - sdk::types::AccessTokenMatcher::Exact(v) => Some(AccessTokenMatcher::Exact(v)), - sdk::types::AccessTokenMatcher::Prefix(v) => Some(AccessTokenMatcher::Prefix(v)), - sdk::types::AccessTokenMatcher::None => None, + sdk::types::AccessTokenMatcher::Exact(v) => AccessTokenMatcher::Exact(v), + sdk::types::AccessTokenMatcher::Prefix(v) => AccessTokenMatcher::Prefix(v), + sdk::types::AccessTokenMatcher::None => AccessTokenMatcher::Prefix(Default::default()), } } } @@ -763,9 +776,9 @@ pub struct AccessTokenScope { impl From for AccessTokenScope { fn from(scope: sdk::types::AccessTokenScope) -> Self { AccessTokenScope { - basins: scope.basins.and_then(BasinMatcher::from_sdk), - streams: scope.streams.and_then(StreamMatcher::from_sdk), - access_tokens: scope.access_tokens.and_then(AccessTokenMatcher::from_sdk), + basins: scope.basins.map(Into::into), + streams: scope.streams.map(Into::into), + access_tokens: scope.access_tokens.map(Into::into), op_group_perms: scope.op_group_perms.map(Into::into), ops: scope.ops.into_iter().map(Operation::from).collect(), } @@ -933,37 +946,6 @@ mod tests { ); } - #[test] - fn match_none_scope_matchers_render_as_no_access() { - colored::control::set_override(false); - - // On the wire, an empty exact name means "match no resources". - let scope: s2_api::v1::access::AccessTokenScope = - serde_json::from_value(serde_json::json!({ - "basins": { "exact": "" }, - "streams": { "exact": "" }, - "access_tokens": { "exact": "" }, - })) - .unwrap(); - let scope = s2_sdk::types::AccessTokenScope::from(scope); - let info = AccessTokenInfo { - id: "tok".to_owned(), - expires_at: None, - auto_prefix_streams: false, - scope: scope.into(), - }; - - assert_eq!( - info.summary_block(3), - "tok expires never\n\ - \x20 basins=∅ streams=∅ tokens=∅ perms=none ops=0" - ); - let json = serde_json::to_value(&info.scope).unwrap(); - assert!(json["basins"].is_null(), "{json}"); - assert!(json["streams"].is_null(), "{json}"); - assert!(json["access_tokens"].is_null(), "{json}"); - } - #[test] fn op_group_perms_summary_omits_empty_groups() { assert_eq!( diff --git a/cli/tests/integration.rs b/cli/tests/integration.rs index 7d0152f0..b193551f 100644 --- a/cli/tests/integration.rs +++ b/cli/tests/integration.rs @@ -563,95 +563,6 @@ fn append_from_stdin() { cleanup_stream(&basin, &stream); } -#[test] -#[serial] -fn append_with_stream_config() { - let basin = unique_name("test-cli-csoa-cfg"); - s2().args([ - "create-basin", - &basin, - "--retention-policy", - "7d", - "--create-stream-on-append", - ]) - .assert() - .success(); - wait_for_basin(&basin); - - let stream = unique_name("test-csoa-new"); - let uri = format!("s2://{basin}/{stream}"); - s2().args([ - "append", - &uri, - "--format", - "text", - "--input", - "-", - "--retention-policy", - "1h", - "--delete-on-empty-min-age", - "5m", - ]) - .write_stdin("first record\n") - .assert() - .success(); - - s2().args(["get-stream-config", &uri]) - .assert() - .success() - .stdout( - predicate::str::contains("1h") - .and(predicate::str::contains("5m")) - .and(predicate::str::contains("7days").not()), - ); - - s2().args([ - "append", - &uri, - "--format", - "text", - "--input", - "-", - "--retention-policy", - "2h", - ]) - .write_stdin("second record\n") - .assert() - .success(); - - s2().args(["get-stream-config", &uri]) - .assert() - .success() - .stdout(predicate::str::contains("1h").and(predicate::str::contains("2h").not())); - - cleanup_stream(&basin, &stream); - cleanup_basin(&basin); -} - -#[test] -#[serial] -fn read_with_stream_config() { - let basin = unique_name("test-cli-csor-cfg"); - s2().args(["create-basin", &basin, "--create-stream-on-read"]) - .assert() - .success(); - wait_for_basin(&basin); - - let stream = unique_name("test-csor-new"); - let uri = format!("s2://{basin}/{stream}"); - s2().args(["read", &uri, "--count", "1", "--retention-policy", "1h"]) - .assert() - .failure(); - - s2().args(["get-stream-config", &uri]) - .assert() - .success() - .stdout(predicate::str::contains("1h")); - - cleanup_stream(&basin, &stream); - cleanup_basin(&basin); -} - #[test] #[serial] fn tail_stream() { diff --git a/cli/tests/lite_wal.rs b/cli/tests/lite_wal.rs deleted file mode 100644 index 609ec24d..00000000 --- a/cli/tests/lite_wal.rs +++ /dev/null @@ -1,227 +0,0 @@ -use std::{ - fs::{self, File}, - io::{Read, Write}, - net::{SocketAddr, TcpListener, TcpStream}, - path::Path, - process::{Child, Command as ProcessCommand, Stdio}, - time::{Duration, Instant}, -}; - -use assert_cmd::Command; -use tempfile::TempDir; - -fn base_command(home: &Path) -> ProcessCommand { - let mut cmd = ProcessCommand::new(assert_cmd::cargo::cargo_bin!("s2")); - cmd.env_clear() - .env("HOME", home) - .env("XDG_CONFIG_HOME", home.join(".config")) - .env("APPDATA", home) - .env("USERPROFILE", home) - .env("NO_COLOR", "1") - .env("RUST_LOG", "warn"); - cmd -} - -fn command(home: &Path) -> Command { - let mut cmd = Command::from_std(base_command(home)); - cmd.timeout(Duration::from_secs(30)); - cmd -} - -#[test] -fn wal_store_requires_a_persistent_main_store_and_one_backend() { - let home = tempfile::tempdir().unwrap(); - for args in [ - vec!["lite", "--wal-bucket", "wal"], - vec!["lite", "--wal-local-root", "wal"], - vec![ - "lite", - "--bucket", - "main", - "--wal-bucket", - "wal", - "--wal-local-root", - "wal", - ], - vec!["lite", "--bucket", "main", "--local-root", "main"], - ] { - command(home.path()).args(args).assert().failure().code(2); - } - command(home.path()) - .env("S2LITE_WAL_BUCKET", "wal") - .args(["lite"]) - .assert() - .failure() - .code(2); -} - -struct Server { - child: Child, - address: SocketAddr, -} - -impl Server { - fn start(root: &Path, attempt: &str) -> Self { - for bind_attempt in 0.. { - let log_path = root.join(format!("server-{attempt}-{bind_attempt}.log")); - match Self::spawn(root, &log_path) { - Ok(server) => return server, - Err(log) if bind_attempt < 5 && log.contains("Address already in use") => { - continue; - } - Err(log) => panic!("Server exited before becoming healthy: {log}"), - } - } - unreachable!() - } - - fn spawn(root: &Path, log_path: &Path) -> Result { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - drop(listener); - let log = File::create(log_path).unwrap(); - let mut cmd = base_command(root); - cmd.args([ - "lite", - "--path", - "db", - "--port", - &address.port().to_string(), - "--local-root", - ]) - .arg(root.join("main")) - .env("S2LITE_WAL_LOCAL_ROOT", root.join("wal")) - .env("SL8_FLUSH_INTERVAL", "1ms") - .env("SL8_L0_SST_SIZE_BYTES", "65536"); - let child = cmd - .stdin(Stdio::null()) - .stdout(log.try_clone().unwrap()) - .stderr(log) - .spawn() - .unwrap(); - let mut server = Self { child, address }; - let deadline = Instant::now() + Duration::from_secs(30); - loop { - if server.child.try_wait().unwrap().is_some() { - return Err(fs::read_to_string(log_path).unwrap()); - } - if let Ok(mut stream) = TcpStream::connect_timeout(&address, Duration::from_millis(100)) - { - stream - .set_read_timeout(Some(Duration::from_secs(1))) - .unwrap(); - stream - .set_write_timeout(Some(Duration::from_secs(1))) - .unwrap(); - let mut response = [0; 256]; - if stream - .write_all( - b"GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", - ) - .is_ok() - && let Ok(n) = stream.read(&mut response) - && response[..n].starts_with(b"HTTP/1.1 200") - { - return Ok(server); - } - } - assert!( - Instant::now() < deadline, - "Server did not become healthy: {}", - fs::read_to_string(log_path).unwrap() - ); - std::thread::sleep(Duration::from_millis(25)); - } - } - - fn client(&self, root: &Path, args: &[&str], input: Option<&str>) -> String { - let endpoint = format!("http://{}", self.address); - let mut cmd = command(root); - cmd.env("S2_ACCOUNT_ENDPOINT", &endpoint) - .env("S2_BASIN_ENDPOINT", &endpoint) - .env("S2_ACCESS_TOKEN", "test") - .args(args); - if let Some(input) = input { - cmd.write_stdin(input); - } - let output = cmd.output().unwrap(); - assert!( - output.status.success(), - "{args:?}: {}", - String::from_utf8_lossy(&output.stderr) - ); - String::from_utf8(output.stdout).unwrap() - } - - fn crash(&mut self) { - self.child.kill().unwrap(); - self.child.wait().unwrap(); - } -} - -impl Drop for Server { - fn drop(&mut self) { - let _ = self.child.kill(); - let _ = self.child.wait(); - } -} - -fn has_sst(path: &Path) -> bool { - fs::read_dir(path).is_ok_and(|entries| { - entries - .flatten() - .any(|entry| entry.path().extension().is_some_and(|ext| ext == "sst")) - }) -} - -#[test] -fn separate_wal_routes_files_and_recovers_acknowledged_records_after_restart() { - let root: TempDir = tempfile::tempdir().unwrap(); - let mut server = Server::start(root.path(), "before"); - server.client(root.path(), &["create-basin", "wal-test"], None); - server.client( - root.path(), - &["create-stream", "s2://wal-test/recovery"], - None, - ); - let mut input = (0..64) - .map(|i| format!("record-{i}-{}\n", "x".repeat(4096))) - .collect::(); - server.client( - root.path(), - &["append", "s2://wal-test/recovery"], - Some(&input), - ); - let deadline = Instant::now() + Duration::from_secs(10); - while !has_sst(&root.path().join("main/db/compacted")) { - assert!(Instant::now() < deadline, "Main-store SST was not flushed"); - std::thread::sleep(Duration::from_millis(25)); - } - // Leave a final small write in the WAL after the earlier data reaches L0. - let last = "last-acknowledged-record\n"; - server.client( - root.path(), - &["append", "s2://wal-test/recovery"], - Some(last), - ); - input.push_str(last); - assert!(has_sst(&root.path().join("wal/db/wal"))); - assert!(!root.path().join("main/db/wal").exists()); - assert!(!root.path().join("wal/db/compacted").exists()); - assert!(!root.path().join("wal/db/manifest").exists()); - server.crash(); - let restarted = Server::start(root.path(), "after"); - let read = restarted.client( - root.path(), - &[ - "read", - "s2://wal-test/recovery", - "--seq-num", - "0", - "--count", - "65", - ], - None, - ); - assert_eq!(read, input); -} diff --git a/common/CHANGELOG.md b/common/CHANGELOG.md index 8fa9a24a..5d7db4c4 100644 --- a/common/CHANGELOG.md +++ b/common/CHANGELOG.md @@ -2,30 +2,6 @@ All notable changes to this project will be documented in this file. -## [0.42.0] - 2026-09-25 - -### Features - -- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) - - - -## [0.41.3] - 2026-09-22 - -### Miscellaneous Tasks - -- Update Cargo.toml dependencies - - - -## [0.41.2] - 2026-09-10 - -### Bug Fixes - -- Reject NUL bytes in stream names and access token IDs ([#728](https://github.com/s2-streamstore/s2/issues/728)) - - - ## [0.41.1] - 2026-07-22 ### Miscellaneous Tasks diff --git a/common/Cargo.toml b/common/Cargo.toml index acdd4354..202963a7 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-common" -version = "0.42.0" +version = "0.41.1" description = "Common stuff for client and servers for S2, the durable streams API" edition.workspace = true license.workspace = true diff --git a/common/src/config.rs b/common/src/config.rs index 8fc17e45..83526ec2 100644 --- a/common/src/config.rs +++ b/common/src/config.rs @@ -2,9 +2,8 @@ //! //! Stream configuration uses three representations: //! -//! - Merged (`StreamConfig`, `TimestampingConfig`, `DeleteOnEmptyConfig`): values produced by -//! merging optional configs with defaults using `merge()`. Storage class remains unspecified when -//! neither the stream nor basin supplies one. +//! - Resolved (`StreamConfig`, `TimestampingConfig`, `DeleteOnEmptyConfig`): concrete values, +//! produced by merging optional configs with defaults using `merge()`. //! //! - Optional (`OptionalStreamConfig`, `OptionalTimestampingConfig`, //! `OptionalDeleteOnEmptyConfig`): partial configuration layers, where `None` means "not set at @@ -18,18 +17,40 @@ //! applies the inner reconfiguration to the existing value, while `Specified(None)` //! clears it to the default. //! -//! `merge()` applies configuration layers with precedence: -//! stream-level → basin-level → field default. +//! `merge()` resolves optional configs into resolved configs with precedence: +//! stream-level → basin-level → system default (via `Option::or` chaining). //! //! Basin config also carries basin-level knobs like `stream_cipher`, //! `create_stream_on_append`, and `create_stream_on_read`. use std::time::Duration; -use compact_str::CompactString; - use crate::{ValidationError, encryption::EncryptionAlgorithm, maybe::Maybe}; +#[derive( + Debug, + Default, + Clone, + Copy, + strum::Display, + strum::IntoStaticStr, + strum::EnumIter, + strum::FromRepr, + strum::EnumString, + PartialEq, + Eq, + Hash, +)] +#[cfg_attr(feature = "clap", derive(clap::ValueEnum))] +#[repr(u8)] +pub enum StorageClass { + #[strum(serialize = "standard")] + Standard = 1, + #[default] + #[strum(serialize = "express")] + Express = 2, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RetentionPolicy { Age(Duration), @@ -89,7 +110,7 @@ impl DeleteOnEmptyConfig { #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct StreamConfig { - pub storage_class: Option, + pub storage_class: StorageClass, pub retention_policy: RetentionPolicy, pub timestamping: TimestampingConfig, pub delete_on_empty: DeleteOnEmptyConfig, @@ -108,7 +129,7 @@ pub struct DeleteOnEmptyReconfiguration { #[derive(Debug, Clone, Default)] pub struct StreamReconfiguration { - pub storage_class: Maybe>, + pub storage_class: Maybe>, pub retention_policy: Maybe>, pub timestamping: Maybe>, pub delete_on_empty: Maybe>, @@ -196,7 +217,7 @@ impl From for OptionalDeleteOnEmptyConfig { #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct OptionalStreamConfig { - pub storage_class: Option, + pub storage_class: Option, pub retention_policy: Option, pub timestamping: OptionalTimestampingConfig, pub delete_on_empty: OptionalDeleteOnEmptyConfig, @@ -237,7 +258,10 @@ impl OptionalStreamConfig { } pub fn merge(self, basin_defaults: Self) -> StreamConfig { - let storage_class = self.storage_class.or(basin_defaults.storage_class); + let storage_class = self + .storage_class + .or(basin_defaults.storage_class) + .unwrap_or_default(); let retention_policy = self .retention_policy @@ -267,7 +291,7 @@ impl From for StreamConfig { } = value; Self { - storage_class, + storage_class: storage_class.unwrap_or_default(), retention_policy: retention_policy.unwrap_or_default(), timestamping: timestamping.into(), delete_on_empty: delete_on_empty.into(), @@ -285,7 +309,7 @@ impl From for OptionalStreamConfig { } = value; Self { - storage_class, + storage_class: Some(storage_class), retention_policy: Some(retention_policy), timestamping: timestamping.into(), delete_on_empty: delete_on_empty.into(), diff --git a/common/src/location.rs b/common/src/location.rs index 86605447..6ce67076 100644 --- a/common/src/location.rs +++ b/common/src/location.rs @@ -146,8 +146,6 @@ impl From for CompactString { pub struct LocationInfo { pub name: LocationName, pub is_private: bool, - pub storage_classes: Vec, - pub default_storage_class: CompactString, } #[cfg(test)] diff --git a/deny.toml b/deny.toml index 24aac6b4..48b28369 100644 --- a/deny.toml +++ b/deny.toml @@ -6,6 +6,8 @@ ignore = [ # Transitive deps via slatedb/foyer — not actionable. "RUSTSEC-2024-0436", # paste unmaintained "RUSTSEC-2025-0141", # bincode unmaintained + # Transitive dep via tabled_derive — no fixed release available. + "RUSTSEC-2026-0173", # proc-macro-error2 unmaintained # Transitive dep via object_store (slatedb) — fix requires object_store >=0.13. "RUSTSEC-2026-0194", # quick-xml quadratic attribute check "RUSTSEC-2026-0195", # quick-xml unbounded namespace allocation diff --git a/lite/CHANGELOG.md b/lite/CHANGELOG.md index 4f80ead3..3d7ff266 100644 --- a/lite/CHANGELOG.md +++ b/lite/CHANGELOG.md @@ -2,81 +2,6 @@ All notable changes to this project will be documented in this file. -## [0.43.0] - 2026-09-25 - -### Features - -- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) - -### Bug Fixes - -- Gate basin deletion draining on per-basin progress ([#779](https://github.com/s2-streamstore/s2/issues/779)) - - - -## [0.42.14] - 2026-09-24 - -### Bug Fixes - -- Coalesce delete-on-empty scheduling ([#772](https://github.com/s2-streamstore/s2/issues/772)) - - - -## [0.42.13] - 2026-09-22 - -### Features - -- Support a separate WAL object store ([#766](https://github.com/s2-streamstore/s2/issues/766)) - -### Bug Fixes - -- Keep streamers alive until pending writes settle ([#757](https://github.com/s2-streamstore/s2/issues/757)) -- Await durable deletion markers on retries ([#756](https://github.com/s2-streamstore/s2/issues/756)) -- Prevent stale or missed stream config updates ([#759](https://github.com/s2-streamstore/s2/issues/759)) -- Apply the at-tail read guard after resolving a timestamp start ([#762](https://github.com/s2-streamstore/s2/issues/762)) -- Prevent deletion races when recreating streams ([#760](https://github.com/s2-streamstore/s2/issues/760)) -- Preserve SSE read budgets across reconnects ([#727](https://github.com/s2-streamstore/s2/issues/727)) - -### Performance - -- Avoid copying serialized append buffers ([#763](https://github.com/s2-streamstore/s2/issues/763)) -- Reuse acknowledgement queue capacity ([#764](https://github.com/s2-streamstore/s2/issues/764)) - - - -## [0.42.12] - 2026-09-16 - -### Miscellaneous Tasks - -- Update Cargo.lock dependencies - - - -## [0.42.11] - 2026-09-11 - -### Features - -- `s2-stream-config` header for auto-created streams ([#718](https://github.com/s2-streamstore/s2/issues/718)) - - - -## [0.42.10] - 2026-09-11 - -### Miscellaneous Tasks - -- Update Cargo.lock dependencies - - - -## [0.42.9] - 2026-09-10 - -### Bug Fixes - -- Close SlateDB on graceful shutdown -- Reject NUL bytes in stream names and access token IDs ([#728](https://github.com/s2-streamstore/s2/issues/728)) - - - ## [0.42.8] - 2026-09-01 ### Miscellaneous Tasks diff --git a/lite/Cargo.toml b/lite/Cargo.toml index 8f0d8988..bb5c7709 100644 --- a/lite/Cargo.toml +++ b/lite/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-lite" -version = "0.43.0" +version = "0.42.8" description = "Lightweight server implementation of S2, the durable streams API, backed by object storage" edition.workspace = true license.workspace = true diff --git a/lite/src/backend/append.rs b/lite/src/backend/append.rs index aa83f752..82343edb 100644 --- a/lite/src/backend/append.rs +++ b/lite/src/backend/append.rs @@ -7,7 +7,6 @@ use std::{ use futures::{Stream, StreamExt as _, future::OptionFuture, stream::FuturesOrdered}; use s2_common::{ basin::BasinName, - config::OptionalStreamConfig, encryption::{EncryptionKey, EncryptionSpec}, record::{SeqNum, StreamPosition}, stream::{AppendAck, AppendInput, StreamName}, @@ -15,26 +14,20 @@ use s2_common::{ use s2_storage::record::encrypt_append_input; use tokio::sync::oneshot; -use super::{Backend, StreamHandle, core::AutoCreateOn}; +use super::{Backend, StreamHandle}; use crate::backend::error::{AppendError, AppendErrorInternal, StorageError}; impl Backend { - /// Open a stream for an append or append session. - /// - /// `stream_config` is applied if the stream is created on append. Unset fields inherit the - /// basin's default stream configuration. Ignored if the stream already exists. pub async fn open_for_append( &self, basin: &BasinName, stream: &StreamName, encryption_key: Option, - stream_config: OptionalStreamConfig, ) -> Result { self.stream_handle_with_auto_create::( basin, stream, - AutoCreateOn::Append, - stream_config, + |config| config.create_stream_on_append, |cipher| Ok(EncryptionSpec::resolve(cipher, encryption_key)?), ) .await @@ -141,8 +134,6 @@ pub fn admit( } } -const MIN_PENDING_APPENDS_CAPACITY: usize = 16; - #[derive(Debug, Default)] pub struct PendingAppends { queue: VecDeque, @@ -197,12 +188,8 @@ impl PendingAppends { } // Lots of small appends could cause this, // as we bound only on total bytes not num batches. - // Keep a small buffer to reuse across ordinary drain/refill cycles. - if self.queue.capacity() > MIN_PENDING_APPENDS_CAPACITY - && self.queue.capacity() >= 4 * self.queue.len() - { - self.queue - .shrink_to((self.queue.len() * 2).max(MIN_PENDING_APPENDS_CAPACITY)); + if self.queue.capacity() >= 4 * self.queue.len() { + self.queue.shrink_to(self.queue.len() * 2); } } @@ -284,115 +271,3 @@ impl BlockedReplySender { let _ = self.tx.send(reply); } } - -#[cfg(test)] -mod tests { - use super::*; - - fn position(seq_num: SeqNum) -> StreamPosition { - StreamPosition { - seq_num, - timestamp: seq_num, - } - } - - fn accept( - pending: &mut PendingAppends, - seq_num: SeqNum, - ) -> oneshot::Receiver> { - let (tx, rx) = oneshot::channel(); - let ticket = admit(tx, None).expect("open receiver"); - pending.accept(ticket, position(seq_num)..position(seq_num + 1)); - rx - } - - #[rstest::rstest] - #[case(1)] - #[case(16)] - fn small_append_bursts_reuse_queue_capacity(#[case] burst_size: SeqNum) { - let mut pending = PendingAppends::new(); - let mut retained_capacity = None; - for burst in 0..4 { - let start = burst * burst_size; - let receivers: Vec<_> = (start..start + burst_size) - .map(|seq_num| accept(&mut pending, seq_num)) - .collect(); - let capacity = *retained_capacity.get_or_insert(pending.queue.capacity()); - assert_eq!(pending.queue.capacity(), capacity); - - // The streamer advances durability one append at a time, even when - // the whole burst became durable in the same WAL flush. - for (offset, mut rx) in receivers.into_iter().enumerate() { - let seq_num = start + offset as SeqNum; - assert!(matches!( - rx.try_recv(), - Err(oneshot::error::TryRecvError::Empty) - )); - pending.on_stable(position(seq_num + 1)); - let ack = rx - .try_recv() - .expect("ack delivered") - .expect("append accepted"); - assert_eq!(ack.start, position(seq_num)); - assert_eq!(ack.end, position(seq_num + 1)); - assert_eq!(ack.tail, position(seq_num + 1)); - assert_eq!(pending.queue.capacity(), capacity); - } - assert!(pending.queue.is_empty()); - } - } - - #[test] - fn unused_queue_stays_unallocated() { - let mut pending = PendingAppends::new(); - pending.on_stable(StreamPosition::MIN); - assert_eq!(pending.queue.capacity(), 0); - } - - #[test] - fn large_append_burst_releases_excess_capacity() { - let mut pending = PendingAppends::new(); - let mut receivers: Vec<_> = (0..128) - .map(|seq_num| accept(&mut pending, seq_num)) - .collect(); - let peak_capacity = pending.queue.capacity(); - - pending.on_stable(position(96)); - assert_eq!(pending.queue.len(), 32); - let reduced_capacity = pending.queue.capacity(); - assert!(reduced_capacity < peak_capacity); - for (seq_num, rx) in receivers.iter_mut().enumerate() { - if seq_num < 96 { - let ack = rx - .try_recv() - .expect("ack delivered") - .expect("append accepted"); - assert_eq!(ack.start, position(seq_num as SeqNum)); - assert_eq!(ack.end, position(seq_num as SeqNum + 1)); - assert_eq!(ack.tail, position(96)); - } else { - assert!(matches!( - rx.try_recv(), - Err(oneshot::error::TryRecvError::Empty) - )); - } - } - - // A small change in occupancy should not immediately shrink again. - pending.on_stable(position(97)); - assert_eq!(pending.queue.capacity(), reduced_capacity); - pending.on_stable(position(128)); - assert!(pending.queue.is_empty()); - assert!(pending.queue.capacity() > 0); - assert!(pending.queue.capacity() <= MIN_PENDING_APPENDS_CAPACITY); - for (seq_num, rx) in receivers.iter_mut().enumerate().skip(96) { - let ack = rx - .try_recv() - .expect("ack delivered") - .expect("append accepted"); - assert_eq!(ack.start, position(seq_num as SeqNum)); - assert_eq!(ack.end, position(seq_num as SeqNum + 1)); - assert_eq!(ack.tail, position(if seq_num == 96 { 97 } else { 128 })); - } - } -} diff --git a/lite/src/backend/basins.rs b/lite/src/backend/basins.rs index 73f1cb01..c37511af 100644 --- a/lite/src/backend/basins.rs +++ b/lite/src/backend/basins.rs @@ -11,11 +11,7 @@ use slatedb::{ }; use time::OffsetDateTime; -use super::{ - Backend, - bgtasks::BgtaskTrigger, - store::{db_txn_commit_durable, db_txn_get, db_txn_get_with}, -}; +use super::{Backend, bgtasks::BgtaskTrigger, store::db_txn_get}; use crate::backend::{ error::{ BasinAlreadyExistsError, BasinDeletionPendingError, BasinNotFoundError, DeleteBasinError, @@ -67,8 +63,6 @@ impl Backend { Ok(Page::new(basins, has_more)) } - /// Any outcome asserting the basin exists — `Created`, `Updated`, `Noop`, - /// or `BasinAlreadyExists` — is readable at `DurabilityLevel::Remote`. pub async fn provision_basin( &self, basin: BasinName, @@ -79,12 +73,7 @@ impl Backend { let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; - // A transaction can see metadata that has not been flushed yet. - let (existing_meta, existing_seq) = db_txn_get_with(&txn, &meta_key, |entry| { - Ok((kv::basin_meta::deser_value(entry.value)?, entry.seq)) - }) - .await? - .unzip(); + let existing_meta = db_txn_get(&txn, &meta_key, kv::basin_meta::deser_value).await?; if let Some(existing_meta) = &existing_meta && existing_meta.deleted_at.is_some() { @@ -96,7 +85,7 @@ impl Backend { let new_creation_idempotency_key = request_token .as_ref() .map(|req_token| creation_idempotency_key(req_token, &config)); - let result = if new_creation_idempotency_key.is_some() + return if new_creation_idempotency_key.is_some() && existing.creation_idempotency_key == new_creation_idempotency_key { Ok(ProvisionResult::Noop(BasinInfo { @@ -108,10 +97,6 @@ impl Backend { } else { Err(BasinAlreadyExistsError { basin }.into()) }; - drop(txn); - self.await_durable_seq(existing_seq.expect("existing meta was read")) - .await?; - return result; } (Some(existing), ProvisionMode::Ensure) => { let meta = kv::basin_meta::BasinMeta { @@ -145,15 +130,11 @@ impl Backend { }), }; - if matches!(&outcome, ProvisionResult::Noop(_)) { - drop(txn); - self.await_durable_seq(existing_seq.expect("noop implies existing meta")) - .await?; - } else { + if !matches!(&outcome, ProvisionResult::Noop(_)) { let meta = outcome.inner(); txn.put(&meta_key, kv::basin_meta::ser_value(meta))?; - db_txn_commit_durable(txn).await?; + txn.commit().await?; } Ok(outcome.map(|meta| BasinInfo { @@ -198,7 +179,7 @@ impl Backend { txn.put(&meta_key, kv::basin_meta::ser_value(&meta))?; - db_txn_commit_durable(txn).await?; + txn.commit().await?; Ok(meta.config) } @@ -206,11 +187,7 @@ impl Backend { pub async fn delete_basin(&self, basin: BasinName) -> Result<(), DeleteBasinError> { let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; let meta_key = kv::basin_meta::ser_key(&basin); - let Some((mut meta, seq)) = db_txn_get_with(&txn, &meta_key, |entry| { - Ok((kv::basin_meta::deser_value(entry.value)?, entry.seq)) - }) - .await? - else { + let Some(mut meta) = db_txn_get(&txn, &meta_key, kv::basin_meta::deser_value).await? else { return Err(BasinNotFoundError { basin }.into()); }; if meta.deleted_at.is_none() { @@ -220,13 +197,9 @@ impl Backend { kv::basin_deletion_pending::ser_key(&basin), kv::basin_deletion_pending::ser_value(&StreamNameStartAfter::default()), )?; - db_txn_commit_durable(txn).await?; - } else { - // A retry may observe the marker before the first delete has flushed. - drop(txn); - self.await_durable_seq(seq).await?; + txn.commit().await?; + self.bgtask_trigger(BgtaskTrigger::BasinDeletion); } - self.bgtask_trigger(BgtaskTrigger::BasinDeletion); Ok(()) } } diff --git a/lite/src/backend/bgtasks/basin_deletion.rs b/lite/src/backend/bgtasks/basin_deletion.rs index 1dbb4344..e11563be 100644 --- a/lite/src/backend/bgtasks/basin_deletion.rs +++ b/lite/src/backend/bgtasks/basin_deletion.rs @@ -10,7 +10,6 @@ use slatedb::{ }; use tracing::instrument; -use super::{ItemProgress, PageProgress}; use crate::backend::{ Backend, error::{BasinDeletionError, ListStreamsError, StorageError}, @@ -26,17 +25,17 @@ impl Backend { if page.values.is_empty() { return Ok(page.has_more); } - let mut progress = PageProgress::new(page.has_more); let mut processed = stream::iter(page.values) .map(|(basin, cursor)| { let backend = self.clone(); async move { backend.process_basin_deletion(basin, cursor).await } }) .buffer_unordered(CONCURRENCY); + let mut has_more = page.has_more; while let Some(result) = processed.next().await { - progress.record(result?); + has_more |= result?; } - Ok(progress.should_continue()) + Ok(has_more) } async fn list_basin_deletion_pending( @@ -69,7 +68,7 @@ impl Backend { &self, basin: BasinName, cursor: StreamNameStartAfter, - ) -> Result { + ) -> Result { let request = ListStreamsRequest { prefix: StreamNamePrefix::default(), start_after: cursor.clone(), @@ -95,21 +94,20 @@ impl Backend { if page.has_more { self.set_basin_deletion_cursor(&basin, &last_stream.expect("non-empty stream page")) .await?; - Ok(ItemProgress::Advanced) + Ok(true) } else if last_stream.is_some() || !cursor.as_ref().is_empty() { // Streams still pending deletion or cursor was advanced past // earlier entries. Reset cursor so the next tick re-scans from - // the beginning. A reset is not progress: counting it would - // rescan a multi-page basin in a tight loop. + // the beginning. if !cursor.as_ref().is_empty() { self.set_basin_deletion_cursor(&basin, &StreamNameStartAfter::default()) .await?; } - Ok(ItemProgress::Blocked) + Ok(false) } else { // No streams from the very beginning — safe to complete. self.complete_basin_deletion(&basin).await?; - Ok(ItemProgress::Completed) + Ok(false) } } @@ -124,7 +122,7 @@ impl Backend { kv::basin_deletion_pending::ser_key(basin), kv::basin_deletion_pending::ser_value(cursor), ); - self.db.write(batch).await?.await_durable().await?; + self.db.write(batch).await?; Ok(()) } @@ -133,7 +131,7 @@ impl Backend { let mut batch = WriteBatch::new(); batch.delete(kv::basin_meta::ser_key(basin)); batch.delete(kv::basin_deletion_pending::ser_key(basin)); - self.db.write(batch).await?.await_durable().await?; + self.db.write(batch).await?; Ok(()) } } @@ -151,7 +149,7 @@ mod tests { use time::OffsetDateTime; use super::super::tests::test_backend; - use crate::backend::{Backend, kv, test_util::DbWriteTestExt as _}; + use crate::backend::{Backend, kv}; fn basin_meta(deleted_at: Option) -> kv::basin_meta::BasinMeta { kv::basin_meta::BasinMeta { @@ -187,16 +185,16 @@ mod tests { kv::basin_meta::ser_key(basin), kv::basin_meta::ser_value(&basin_meta(Some(OffsetDateTime::now_utc()))), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( kv::basin_deletion_pending::ser_key(basin), kv::basin_deletion_pending::ser_value(&StreamNameStartAfter::default()), ) - .assert_durable() - .await; + .await + .unwrap(); } async fn seed_tombstoned_streams(backend: &Backend, basin: &BasinName, count: usize) { @@ -209,7 +207,7 @@ mod tests { kv::stream_meta::ser_value(&stream_meta(Some(deleted_at))), ); } - backend.db.write(batch).assert_durable().await; + backend.db.write(batch).await.unwrap(); } #[tokio::test] @@ -223,16 +221,16 @@ mod tests { kv::basin_meta::ser_key(&basin), kv::basin_meta::ser_value(&basin_meta(Some(OffsetDateTime::now_utc()))), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( kv::basin_deletion_pending::ser_key(&basin), kv::basin_deletion_pending::ser_value(&StreamNameStartAfter::default()), ) - .assert_durable() - .await; + .await + .unwrap(); let has_more = backend.clone().tick_basin_deletion().await.unwrap(); assert!(!has_more); @@ -268,17 +266,8 @@ mod tests { kv::stream_meta::ser_key(&basin, &stream), kv::stream_meta::ser_value(&stream_meta(None)), ) - .assert_durable() - .await; - - backend - .db - .put( - kv::stream_id_mapping::ser_key(crate::stream_id::StreamId::new(&basin, &stream)), - kv::stream_id_mapping::ser_value(&basin, &stream), - ) - .assert_durable() - .await; + .await + .unwrap(); let has_more = backend.clone().tick_basin_deletion().await.unwrap(); assert!(!has_more); @@ -395,16 +384,16 @@ mod tests { kv::basin_meta::ser_key(&basin), kv::basin_meta::ser_value(&basin_meta(Some(OffsetDateTime::now_utc()))), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( kv::basin_deletion_pending::ser_key(&basin), kv::basin_deletion_pending::ser_value(&cursor), ) - .assert_durable() - .await; + .await + .unwrap(); // First tick resets cursor from past-end back to the beginning. let has_more = backend.clone().tick_basin_deletion().await.unwrap(); @@ -446,8 +435,8 @@ mod tests { kv::stream_meta::ser_key(&basin, &stream), kv::stream_meta::ser_value(&stream_meta(Some(deleted_at))), ) - .assert_durable() - .await; + .await + .unwrap(); let has_more = backend.clone().tick_basin_deletion().await.unwrap(); assert!(!has_more); @@ -485,8 +474,8 @@ mod tests { kv::stream_meta::ser_key(&basin, &stream), kv::stream_meta::ser_value(&stream_meta(Some(deleted_at))), ) - .assert_durable() - .await; + .await + .unwrap(); // First tick: blocked by tombstoned stream. let has_more = backend.clone().tick_basin_deletion().await.unwrap(); @@ -504,8 +493,8 @@ mod tests { backend .db .delete(kv::stream_meta::ser_key(&basin, &stream)) - .assert_durable() - .await; + .await + .unwrap(); // Second tick: no streams remain, completes. let has_more = backend.clone().tick_basin_deletion().await.unwrap(); diff --git a/lite/src/backend/bgtasks/mod.rs b/lite/src/backend/bgtasks/mod.rs index ed219387..a92916f8 100644 --- a/lite/src/backend/bgtasks/mod.rs +++ b/lite/src/backend/bgtasks/mod.rs @@ -9,63 +9,6 @@ mod basin_deletion; mod stream_doe; mod stream_trim; -/// Keep draining the backlog while at least one item makes progress. A page where -/// every item is blocked waits for the next tick instead of retrying in a tight loop. -struct PageProgress { - has_more: bool, - any_progressed: bool, -} - -enum ItemProgress { - /// Made progress, and more work for this item can run immediately. - Advanced, - Completed, - /// Made no progress; the item waits for a later tick. - Blocked, -} - -impl PageProgress { - fn new(has_more: bool) -> Self { - Self { - has_more, - any_progressed: false, - } - } - - fn record(&mut self, item: ItemProgress) { - match item { - ItemProgress::Advanced => { - self.has_more = true; - self.any_progressed = true; - } - ItemProgress::Completed => self.any_progressed = true, - ItemProgress::Blocked => {} - } - } - - fn record_result( - &mut self, - result: Result, - is_conflict: fn(&E) -> bool, - ) -> Result, E> { - match result { - Ok(value) => { - self.record(ItemProgress::Completed); - Ok(Some(value)) - } - Err(err) if is_conflict(&err) => { - self.record(ItemProgress::Blocked); - Ok(None) - } - Err(err) => Err(err), - } - } - - fn should_continue(&self) -> bool { - self.has_more && self.any_progressed - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum BgtaskTrigger { BasinDeletion, @@ -229,13 +172,4 @@ mod tests { assert_eq!(calls.load(Ordering::SeqCst), 3); } - - #[test] - fn blocked_page_waits_for_next_tick() { - let mut progress = PageProgress::new(true); - progress.record(ItemProgress::Blocked); - assert!(!progress.should_continue()); - progress.record(ItemProgress::Completed); - assert!(progress.should_continue()); - } } diff --git a/lite/src/backend/bgtasks/stream_doe.rs b/lite/src/backend/bgtasks/stream_doe.rs index d5c3a19d..d09b108d 100644 --- a/lite/src/backend/bgtasks/stream_doe.rs +++ b/lite/src/backend/bgtasks/stream_doe.rs @@ -1,21 +1,21 @@ -use bytes::Bytes; +use std::time::Duration; + use futures::{StreamExt, stream}; use indexmap::IndexMap; -use s2_common::{basin::BasinName, resources::Page, stream::StreamName}; +use itertools::Itertools; +use s2_common::resources::Page; use slatedb::{ - IsolationLevel, WriteBatch, + WriteBatch, config::{DurabilityLevel, ScanOptions}, }; +use tracing::instrument; -use super::PageProgress; use crate::{ backend::{ - Backend, doe, + Backend, error::{DeleteStreamError, StorageError, StreamDeleteOnEmptyError}, - kv, - store::{db_snapshot_get_with, db_txn_commit_durable, db_txn_get, db_txn_get_with}, - streamer::{TerminalTrimCondition, TerminalTrimOutcome}, - timestamp::TimestampSecs, + kv::{self, timestamp::TimestampSecs}, + streamer::{TerminalTrimCondition, doe_arm_delay}, }, stream_id::StreamId, }; @@ -23,1222 +23,780 @@ use crate::{ const PENDING_LIST_LIMIT: usize = 10_000; const CONCURRENCY: usize = 4; -#[derive(Clone, Copy, Debug)] -struct PendingCheck { - stream_id: StreamId, - check: kv::stream_doe_state::Check, +#[derive(Debug)] +struct PendingDoeBatch { + entries: Vec, + last_write_cutoff: Option, } -struct CheckSnapshot { - revision: u64, - creation_seq: u64, - config_seq: u64, - basin: BasinName, - stream: StreamName, +impl PendingDoeBatch { + fn new(entries: Vec) -> Self { + let last_write_cutoff = entries + .iter() + .filter_map(|entry| entry.last_write_cutoff()) + .max(); + Self { + entries, + last_write_cutoff, + } + } + + fn entries(&self) -> &[kv::stream_doe_deadline::Entry] { + &self.entries + } } impl Backend { pub(super) async fn tick_stream_doe(self) -> Result { - // Drain legacy keys regardless of their old deadlines. Their timestamps - // and values have no role in the new scheduler or deletion eligibility. - let mut progress = self.migrate_stream_doe().await?; - let page = self.list_pending_stream_doe(TimestampSecs::now()).await?; - progress.has_more |= page.has_more; + let now = TimestampSecs::now(); + let page = self.list_pending_stream_doe(now).await?; + if page.values.is_empty() { + return Ok(page.has_more); + } let mut processed = stream::iter(page.values) - .map(|pending| { + .map(|(stream_id, pending)| { let backend = self.clone(); - async move { backend.process_stream_doe(pending).await } + async move { backend.process_stream_doe(stream_id, pending).await } }) .buffer_unordered(CONCURRENCY); while let Some(result) = processed.next().await { - progress.record_result(result, StreamDeleteOnEmptyError::is_transaction_conflict)?; + result?; } - Ok(progress.should_continue()) + Ok(page.has_more) } async fn list_pending_stream_doe( &self, now: TimestampSecs, - ) -> Result, StorageError> { + ) -> Result, StorageError> { let scan_opts = ScanOptions { durability_filter: DurabilityLevel::Remote, ..Default::default() }; let mut it = self .db - .scan_with_options(kv::stream_doe_check::due_key_range(now), &scan_opts) + .scan_with_options(kv::stream_doe_deadline::expired_key_range(now), &scan_opts) .await?; - let mut pending = Vec::new(); + let mut pending: IndexMap> = IndexMap::new(); + let mut has_more = false; + let mut count = 0; while let Some(kv) = it.next().await? { - let (stream_id, check) = kv::stream_doe_check::deser_key(kv.key)?; - pending.push(PendingCheck { stream_id, check }); - if pending.len() == PENDING_LIST_LIMIT { - return Ok(Page::new(pending, true)); + let (deadline, stream_id) = kv::stream_doe_deadline::deser_key(kv.key)?; + let min_age = kv::stream_doe_deadline::deser_value(kv.value)?; + assert!(deadline <= now); + pending + .entry(stream_id) + .or_default() + .push(kv::stream_doe_deadline::Entry { deadline, min_age }); + count += 1; + if count == PENDING_LIST_LIMIT { + has_more = true; + break; } } - Ok(Page::new(pending, false)) + Ok(Page::new( + pending + .into_iter() + .map(|(stream_id, entries)| (stream_id, PendingDoeBatch::new(entries))) + .collect_vec(), + has_more, + )) } async fn process_stream_doe( &self, - pending: PendingCheck, + stream_id: StreamId, + pending: PendingDoeBatch, ) -> Result<(), StreamDeleteOnEmptyError> { - let Some(snapshot) = self.observe_doe_check(pending).await? else { - return Ok(()); - }; - let outcome = match self - .delete_stream_with_condition( - snapshot.basin.clone(), - snapshot.stream.clone(), - TerminalTrimCondition::DeleteOnEmpty { - expected_stream_creation_seq: snapshot.creation_seq, - expected_config_seq: snapshot.config_seq, - }, - ) - .await + if let Some(last_write_cutoff) = pending.last_write_cutoff + && let Some((basin, stream)) = self.stream_id_mapping(stream_id).await? { - Ok(outcome) => outcome, - Err(DeleteStreamError::StreamNotFound(_)) => TerminalTrimOutcome::Obsolete, - Err(err) => return Err(err.into()), - }; - self.finish_doe_check(pending, snapshot, outcome).await?; + match self + .delete_stream_with_condition( + basin, + stream, + TerminalTrimCondition::DeleteOnEmpty { last_write_cutoff }, + ) + .await + { + Ok(()) | Err(DeleteStreamError::StreamNotFound(_)) => {} + Err(err) => return Err(err.into()), + } + } + self.clear_doe_deadlines(stream_id, pending.entries()) + .await?; Ok(()) } - /// Capture the scheduler revision before the streamer's asynchronous scan. - async fn observe_doe_check( + #[instrument(ret, err, skip(self, pending), fields(num_deadlines = pending.len()))] + async fn clear_doe_deadlines( &self, - pending: PendingCheck, - ) -> Result, StorageError> { - // Observation needs a consistent view, but no transaction read set. The - // completion transaction validates this revision after the actor's scan. - let snapshot = self.db.snapshot().await?; - let state = db_snapshot_get_with( - &snapshot, - kv::stream_doe_state::ser_key(pending.stream_id), - |entry| Ok((kv::stream_doe_state::deser_value(entry.value)?, entry.seq)), - ) - .await?; - let Some((kv::stream_doe_state::State::Scheduled(_), revision)) = state - .filter(|(state, _)| *state == kv::stream_doe_state::State::Scheduled(pending.check)) - else { - drop(snapshot); - self.discard_doe_check(pending, state).await?; - return Ok(None); + stream_id: StreamId, + pending: &[kv::stream_doe_deadline::Entry], + ) -> Result<(), StorageError> { + let mut batch = WriteBatch::new(); + for entry in pending { + batch.delete(kv::stream_doe_deadline::ser_key(entry.deadline, stream_id)); + } + self.db.write(batch).await?; + Ok(()) + } + + pub(super) async fn arm_doe_on_full_trim( + &self, + stream_id: StreamId, + ) -> Result<(), StorageError> { + let Some((basin, stream)) = self.stream_id_mapping(stream_id).await? else { + return Ok(()); }; - let mapping = db_snapshot_get_with( - &snapshot, - kv::stream_id_mapping::ser_key(pending.stream_id), - |entry| Ok((kv::stream_id_mapping::deser_value(entry.value)?, entry.seq)), - ) - .await?; - if let Some(((basin, stream), creation_seq)) = mapping - && revision >= creation_seq - && let Some((meta, config_seq)) = db_snapshot_get_with( - &snapshot, - kv::stream_meta::ser_key(&basin, &stream), - |entry| Ok((kv::stream_meta::deser_value(entry.value)?, entry.seq)), + let Some(meta) = self + .db_get( + &kv::stream_meta::ser_key(&basin, &stream), + kv::stream_meta::deser_value, ) .await? - && meta.deleted_at.is_none() - && meta.config.delete_on_empty.min_age().is_some() - { - return Ok(Some(CheckSnapshot { - revision, - creation_seq, - config_seq, - basin, - stream, - })); - } - drop(snapshot); - self.discard_doe_check(pending, state).await?; - Ok(None) - } - - /// Obsolete work needs a transaction only when it is actually discarded. - /// Revalidate the snapshot so cleanup cannot erase a concurrent wake. - async fn discard_doe_check( - &self, - pending: PendingCheck, - observed: Option<(kv::stream_doe_state::State, u64)>, - ) -> Result<(), StorageError> { - let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; - let current = db_txn_get_with( - &txn, - kv::stream_doe_state::ser_key(pending.stream_id), - |entry| Ok((kv::stream_doe_state::deser_value(entry.value)?, entry.seq)), - ) - .await?; - if current != observed { + else { + return Ok(()); + }; + if meta.deleted_at.is_some() { return Ok(()); } - match current { - Some((kv::stream_doe_state::State::Scheduled(check), _)) if check == pending.check => { - doe::replace( - &txn, - pending.stream_id, - Some(kv::stream_doe_state::State::Scheduled(check)), - None, - )?; - } - _ => { - txn.delete(kv::stream_doe_check::ser_key( - pending.stream_id, - pending.check, - ))?; - } - } - db_txn_commit_durable(txn).await?; - Ok(()) - } - - async fn finish_doe_check( - &self, - pending: PendingCheck, - snapshot: CheckSnapshot, - outcome: TerminalTrimOutcome, - ) -> Result<(), StorageError> { - let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; - let state = db_txn_get_with( - &txn, - kv::stream_doe_state::ser_key(pending.stream_id), - |entry| Ok((kv::stream_doe_state::deser_value(entry.value)?, entry.seq)), - ) - .await?; - if state - != Some(( - kv::stream_doe_state::State::Scheduled(pending.check), - snapshot.revision, - )) - { - // A trim/configuration event owns the next check, even if it kept the - // same ticket. Consuming or postponing it here would lose that wake. + let Some(min_age) = meta.config.delete_on_empty.min_age() else { return Ok(()); - } - // Successful deletion marks metadata and advances its sequence. Deferred - // outcomes must still belong to the configuration that was inspected. - if outcome != TerminalTrimOutcome::DeletionPending { - let config_seq = db_txn_get_with( - &txn, - kv::stream_meta::ser_key(&snapshot.basin, &snapshot.stream), - |entry| Ok(entry.seq), + }; + let deadline = TimestampSecs::after(doe_arm_delay(Duration::ZERO, min_age)); + self.db + .put( + kv::stream_doe_deadline::ser_key(deadline, stream_id), + kv::stream_doe_deadline::ser_value(min_age), ) .await?; - if config_seq != Some(snapshot.config_seq) { - return Ok(()); - } - } - let next = match outcome { - TerminalTrimOutcome::RetryAt(at) => Some(kv::stream_doe_state::State::Scheduled( - kv::stream_doe_state::Check { - at, - id: rand::random(), - }, - )), - TerminalTrimOutcome::Parked => Some(kv::stream_doe_state::State::Parked), - TerminalTrimOutcome::DeletionPending | TerminalTrimOutcome::Obsolete => None, - }; - // Consuming the old check and installing its successor is one durable - // transaction, including when the caller disappears during commit. - doe::replace( - &txn, - pending.stream_id, - Some(kv::stream_doe_state::State::Scheduled(pending.check)), - next, - )?; - db_txn_commit_durable(txn).await?; Ok(()) } +} - async fn migrate_stream_doe(&self) -> Result { +#[cfg(test)] +mod tests { + use std::{str::FromStr, time::Duration}; + + use s2_common::{ + basin::BasinName, + config::{ + BasinConfig, DeleteOnEmptyReconfiguration, OptionalStreamConfig, RetentionPolicy, + StreamReconfiguration, + }, + maybe::Maybe, + record::StreamPosition, + stream::StreamName, + }; + use slatedb::config::{DurabilityLevel, ScanOptions}; + use time::OffsetDateTime; + + use super::{super::tests::test_backend, PendingDoeBatch, TimestampSecs}; + use crate::{ + backend::{Backend, kv}, + stream_id::StreamId, + }; + + const MIN_AGE: Duration = Duration::from_secs(60); + + fn stream_meta_with_config( + config: OptionalStreamConfig, + created_at: OffsetDateTime, + ) -> kv::stream_meta::StreamMeta { + kv::stream_meta::StreamMeta { + config: config.into(), + cipher: None, + created_at, + deleted_at: None, + creation_idempotency_key: None, + } + } + + fn stream_meta_with_doe_min_age(min_age: Duration) -> kv::stream_meta::StreamMeta { + let mut config = OptionalStreamConfig::default(); + config.delete_on_empty.min_age = Some(min_age); + stream_meta_with_config(config, OffsetDateTime::now_utc()) + } + + async fn seed_stream_with_meta( + backend: &Backend, + basin: &BasinName, + stream: &StreamName, + meta: kv::stream_meta::StreamMeta, + ) -> StreamId { + let stream_id = StreamId::new(basin, stream); + backend + .db + .put( + kv::basin_meta::ser_key(basin), + kv::basin_meta::ser_value(&kv::basin_meta::BasinMeta { + config: BasinConfig::default(), + created_at: OffsetDateTime::now_utc(), + deleted_at: None, + creation_idempotency_key: None, + }), + ) + .await + .unwrap(); + backend + .db + .put( + kv::stream_meta::ser_key(basin, stream), + kv::stream_meta::ser_value(&meta), + ) + .await + .unwrap(); + backend + .db + .put( + kv::stream_id_mapping::ser_key(stream_id), + kv::stream_id_mapping::ser_value(basin, stream), + ) + .await + .unwrap(); + stream_id + } + + async fn list_doe_entries(backend: &Backend) -> Vec<(TimestampSecs, StreamId, Duration)> { let scan_opts = ScanOptions { durability_filter: DurabilityLevel::Remote, ..Default::default() }; - let mut it = self + let mut it = backend .db .scan_with_options( kv::key_type_range(kv::KeyType::StreamDeleteOnEmptyDeadline), &scan_opts, ) - .await?; - let mut pending: IndexMap> = IndexMap::new(); - let mut count = 0; - while let Some(entry) = it.next().await? { - let (_, stream_id, _) = kv::stream_doe_deadline::deser_key(entry.key.clone())?; - pending.entry(stream_id).or_default().push(entry.key); - count += 1; - if count == PENDING_LIST_LIMIT { - break; - } + .await + .unwrap(); + let mut entries = Vec::new(); + while let Some(kv) = it.next().await.unwrap() { + let (deadline, stream_id) = kv::stream_doe_deadline::deser_key(kv.key).unwrap(); + let min_age = kv::stream_doe_deadline::deser_value(kv.value).unwrap(); + entries.push((deadline, stream_id, min_age)); } - let mut progress = PageProgress::new(count == PENDING_LIST_LIMIT); - if pending.is_empty() { - return Ok(progress); - } - let snapshot = self.db.snapshot().await?; - let mut migrations = stream::iter(pending) - .map(|(stream_id, keys)| self.migrate_doe_deadlines(&snapshot, stream_id, keys)) - .buffer_unordered(CONCURRENCY); - let mut cleanup = WriteBatch::new(); - while let Some(result) = migrations.next().await { - if let Some(keys) = - progress.record_result(result, StorageError::is_transaction_conflict)? - { - for key in keys { - cleanup.delete(key); - } - } - } - if !cleanup.is_empty() { - // Later enablement/recreation installs its own schedule. Only - // legacy keys are removed; new state and its revision stay intact. - // Durability also covers any commits observed by the snapshot. - self.db.write(cleanup).await?.await_durable().await?; - } - Ok(progress) + entries } - async fn migrate_doe_deadlines( - &self, - snapshot: &slatedb::DbSnapshot, + async fn put_tail_position( + backend: &Backend, stream_id: StreamId, - keys: Vec, - ) -> Result, StorageError> { - if needs_doe_migration(snapshot, stream_id).await? { - // Initialization and removal remain atomic. Recheck all eligibility - // in the transaction after the snapshot. - self.initialize_doe_from_deadlines(stream_id, keys).await?; - Ok(Vec::new()) - } else { - Ok(keys) - } + position: StreamPosition, + ) -> TimestampSecs { + let key = kv::stream_tail_position::ser_key(stream_id); + backend + .db + .put(key.clone(), kv::stream_tail_position::ser_value(position)) + .await + .unwrap(); + let kv = backend + .db + .get_key_value(key) + .await + .unwrap() + .expect("tail position should exist"); + TimestampSecs::from_millis(kv.create_ts) } - async fn initialize_doe_from_deadlines( - &self, + fn deadline_after(write_timestamp: TimestampSecs, age: Duration) -> TimestampSecs { + let deadline_secs = u64::from(write_timestamp.as_u32()) + .saturating_add(age.as_secs()) + .min(u64::from(u32::MAX)) as u32; + TimestampSecs::from_secs(deadline_secs) + } + + async fn process_pending_stream_doe_at( + backend: &Backend, stream_id: StreamId, - keys: Vec, - ) -> Result<(), StorageError> { - let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; - let mapping = db_txn_get_with(&txn, kv::stream_id_mapping::ser_key(stream_id), |entry| { - Ok((kv::stream_id_mapping::deser_value(entry.value)?, entry.seq)) - }) - .await?; - if let Some(((basin, stream), creation_seq)) = mapping - && let Some(meta) = db_txn_get( - &txn, - kv::stream_meta::ser_key(&basin, &stream), - kv::stream_meta::deser_value, + now: TimestampSecs, + ) { + let mut page = backend.list_pending_stream_doe(now).await.unwrap(); + assert!(!page.has_more); + assert_eq!(page.values.len(), 1); + let (pending_stream_id, pending) = page.values.pop().unwrap(); + assert_eq!(pending_stream_id, stream_id); + + backend + .process_stream_doe(stream_id, pending) + .await + .unwrap(); + } + + #[tokio::test] + async fn stream_doe_marks_deleted_and_clears_deadline() { + let backend = test_backend().await; + let basin = BasinName::from_str("doe-basin").unwrap(); + let stream = StreamName::from_str("doe-stream").unwrap(); + let stream_id = seed_stream_with_meta( + &backend, + &basin, + &stream, + stream_meta_with_doe_min_age(MIN_AGE), + ) + .await; + let write_timestamp = put_tail_position( + &backend, + stream_id, + StreamPosition { + seq_num: 1, + timestamp: 1234, + }, + ) + .await; + let deadline = deadline_after(write_timestamp, MIN_AGE); + + backend + .db + .put( + kv::stream_doe_deadline::ser_key(deadline, stream_id), + kv::stream_doe_deadline::ser_value(MIN_AGE), ) - .await? - && meta.deleted_at.is_none() - && meta.config.delete_on_empty.min_age().is_some() - { - let state = db_txn_get_with(&txn, kv::stream_doe_state::ser_key(stream_id), |entry| { - Ok((kv::stream_doe_state::deser_value(entry.value)?, entry.seq)) - }) - .await?; - if state.is_none_or(|(_, seq)| seq < creation_seq) { - let next = kv::stream_doe_state::State::Scheduled(kv::stream_doe_state::Check { - at: TimestampSecs::now(), - id: rand::random(), - }); - doe::replace(&txn, stream_id, state.map(|(state, _)| state), Some(next))?; - } - // Existing state, including Parked, is deliberately untouched. - } - for key in keys { - txn.delete(key)?; - } - db_txn_commit_durable(txn).await?; - Ok(()) + .await + .unwrap(); + + process_pending_stream_doe_at(&backend, stream_id, deadline).await; + + let meta = backend + .db + .get(kv::stream_meta::ser_key(&basin, &stream)) + .await + .unwrap() + .expect("stream meta should remain"); + let decoded = kv::stream_meta::deser_value(meta).unwrap(); + assert!(decoded.deleted_at.is_some()); + + let deadline_key = backend + .db + .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) + .await + .unwrap(); + assert!(deadline_key.is_none()); } -} -async fn needs_doe_migration( - snapshot: &slatedb::DbSnapshot, - stream_id: StreamId, -) -> Result { - let Some(((basin, stream), creation_seq)) = db_snapshot_get_with( - snapshot, - kv::stream_id_mapping::ser_key(stream_id), - |entry| Ok((kv::stream_id_mapping::deser_value(entry.value)?, entry.seq)), - ) - .await? - else { - return Ok(false); - }; - let Some(meta) = db_snapshot_get_with( - snapshot, - kv::stream_meta::ser_key(&basin, &stream), - |entry| kv::stream_meta::deser_value(entry.value), - ) - .await? - else { - return Ok(false); - }; - if meta.deleted_at.is_some() || meta.config.delete_on_empty.min_age().is_none() { - return Ok(false); + #[tokio::test] + async fn stream_doe_deletes_never_written_stream() { + let backend = test_backend().await; + let basin = BasinName::from_str("doe-basin-never").unwrap(); + let stream = StreamName::from_str("doe-stream-never").unwrap(); + let stream_id = StreamId::new(&basin, &stream); + let min_age = MIN_AGE; + let meta = stream_meta_with_doe_min_age(min_age); + + seed_stream_with_meta(&backend, &basin, &stream, meta).await; + + let write_timestamp = put_tail_position(&backend, stream_id, StreamPosition::MIN).await; + let deadline = deadline_after(write_timestamp, min_age); + backend + .db + .put( + kv::stream_doe_deadline::ser_key(deadline, stream_id), + kv::stream_doe_deadline::ser_value(min_age), + ) + .await + .unwrap(); + + process_pending_stream_doe_at(&backend, stream_id, deadline).await; + + let meta = backend + .db + .get(kv::stream_meta::ser_key(&basin, &stream)) + .await + .unwrap() + .expect("stream meta should remain"); + let decoded = kv::stream_meta::deser_value(meta).unwrap(); + assert!(decoded.deleted_at.is_some()); + + let deadline_key = backend + .db + .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) + .await + .unwrap(); + assert!(deadline_key.is_none()); } - let state_seq = db_snapshot_get_with( - snapshot, - kv::stream_doe_state::ser_key(stream_id), - |entry| { - kv::stream_doe_state::deser_value(entry.value)?; - Ok(entry.seq) - }, - ) - .await?; - Ok(state_seq.is_none_or(|seq| seq < creation_seq)) -} -#[cfg(test)] -mod tests { - use std::time::Duration; + #[tokio::test] + async fn stream_doe_skips_recent_tail_write() { + let backend = test_backend().await; + let basin = BasinName::from_str("doe-basin-recent").unwrap(); + let stream = StreamName::from_str("doe-stream-recent").unwrap(); + let stream_id = seed_stream_with_meta( + &backend, + &basin, + &stream, + stream_meta_with_doe_min_age(MIN_AGE), + ) + .await; + let write_timestamp = put_tail_position( + &backend, + stream_id, + StreamPosition { + seq_num: 1, + timestamp: 1234, + }, + ) + .await; + let deadline = write_timestamp; - use s2_common::{ - config::{ - DeleteOnEmptyReconfiguration, OptionalStreamConfig, RetentionPolicy, - StreamReconfiguration, - }, - maybe::Maybe, - record::{CommandRecord, MeteredExt as _, Record}, - resources::ProvisionMode, - stream::{AppendInput, AppendRecord, AppendRecordParts}, - }; + backend + .db + .put( + kv::stream_doe_deadline::ser_key(deadline, stream_id), + kv::stream_doe_deadline::ser_value(MIN_AGE), + ) + .await + .unwrap(); - use super::*; - use crate::backend::{ - bgtasks::{run_tick, tests::test_backend}, - test_util::{DbWriteTestExt as _, create_stream}, - }; + process_pending_stream_doe_at(&backend, stream_id, deadline).await; - fn config(min_age: u64, retention: RetentionPolicy) -> OptionalStreamConfig { - let mut config = OptionalStreamConfig { - retention_policy: Some(retention), + let meta = backend + .db + .get(kv::stream_meta::ser_key(&basin, &stream)) + .await + .unwrap() + .expect("stream meta should remain"); + let decoded = kv::stream_meta::deser_value(meta).unwrap(); + assert!(decoded.deleted_at.is_none()); + + let deadline_key = backend + .db + .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) + .await + .unwrap(); + assert!(deadline_key.is_none()); + } + + #[tokio::test] + async fn stream_doe_skips_stream_with_records() { + let backend = test_backend().await; + let basin = BasinName::from_str("doe-basin-nonempty").unwrap(); + let stream = StreamName::from_str("doe-stream-nonempty").unwrap(); + let stream_id = seed_stream_with_meta( + &backend, + &basin, + &stream, + stream_meta_with_doe_min_age(MIN_AGE), + ) + .await; + let deadline = TimestampSecs::now(); + + let pos = StreamPosition { + seq_num: 1, + timestamp: 1234, + }; + backend + .db + .put( + kv::stream_record_timestamp::ser_key(stream_id, pos), + kv::stream_record_timestamp::ser_value(), + ) + .await + .unwrap(); + backend + .db + .put( + kv::stream_doe_deadline::ser_key(deadline, stream_id), + kv::stream_doe_deadline::ser_value(MIN_AGE), + ) + .await + .unwrap(); + + let has_more = backend.clone().tick_stream_doe().await.unwrap(); + assert!(!has_more); + + let meta = backend + .db + .get(kv::stream_meta::ser_key(&basin, &stream)) + .await + .unwrap() + .expect("stream meta should remain"); + let decoded = kv::stream_meta::deser_value(meta).unwrap(); + assert!(decoded.deleted_at.is_none()); + + let deadline_key = backend + .db + .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) + .await + .unwrap(); + assert!(deadline_key.is_none()); + + let timestamp_key = backend + .db + .get(kv::stream_record_timestamp::ser_key(stream_id, pos)) + .await + .unwrap(); + assert!(timestamp_key.is_some()); + } + + #[tokio::test] + async fn stream_doe_ignores_future_deadline() { + let backend = test_backend().await; + let basin = BasinName::from_str("doe-basin-future").unwrap(); + let stream = StreamName::from_str("doe-stream-future").unwrap(); + let stream_id = seed_stream_with_meta( + &backend, + &basin, + &stream, + stream_meta_with_doe_min_age(MIN_AGE), + ) + .await; + let deadline = TimestampSecs::after(Duration::from_secs(3600)); + + backend + .db + .put( + kv::stream_doe_deadline::ser_key(deadline, stream_id), + kv::stream_doe_deadline::ser_value(MIN_AGE), + ) + .await + .unwrap(); + + let has_more = backend.clone().tick_stream_doe().await.unwrap(); + assert!(!has_more); + + let meta = backend + .db + .get(kv::stream_meta::ser_key(&basin, &stream)) + .await + .unwrap() + .expect("stream meta should remain"); + let decoded = kv::stream_meta::deser_value(meta).unwrap(); + assert!(decoded.deleted_at.is_none()); + + let deadline_key = backend + .db + .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) + .await + .unwrap(); + assert!(deadline_key.is_some()); + } + + #[tokio::test] + async fn stream_doe_groups_multiple_deadlines() { + let backend = test_backend().await; + let basin = BasinName::from_str("doe-basin-multi").unwrap(); + let stream = StreamName::from_str("doe-stream-multi").unwrap(); + let stream_id = seed_stream_with_meta( + &backend, + &basin, + &stream, + stream_meta_with_doe_min_age(MIN_AGE), + ) + .await; + let far_future = TimestampSecs::after(Duration::from_secs(3600)); + let deadline_a = TimestampSecs::after(Duration::ZERO); + let deadline_b = TimestampSecs::after(Duration::from_secs(1)); + + backend + .db + .put( + kv::stream_doe_deadline::ser_key(deadline_a, stream_id), + kv::stream_doe_deadline::ser_value(MIN_AGE), + ) + .await + .unwrap(); + backend + .db + .put( + kv::stream_doe_deadline::ser_key(deadline_b, stream_id), + kv::stream_doe_deadline::ser_value(MIN_AGE), + ) + .await + .unwrap(); + + let page = backend.list_pending_stream_doe(far_future).await.unwrap(); + assert!(!page.has_more); + assert_eq!(page.values.len(), 1); + let (pending_stream_id, pending) = page.values.into_iter().next().unwrap(); + assert_eq!(pending_stream_id, stream_id); + let mut deadlines: Vec<_> = pending + .entries() + .iter() + .map(|entry| entry.deadline) + .collect(); + deadlines.sort(); + let mut expected = vec![deadline_a, deadline_b]; + expected.sort(); + assert_eq!(deadlines, expected); + + backend + .process_stream_doe(stream_id, pending) + .await + .unwrap(); + + for deadline in [deadline_a, deadline_b] { + let deadline_key = backend + .db + .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) + .await + .unwrap(); + assert!(deadline_key.is_none()); + } + } + + #[tokio::test] + async fn stream_doe_uses_latest_eligible_cutoff_across_pending_entries() { + let backend = test_backend().await; + let basin = BasinName::from_str("doe-basin-pairs").unwrap(); + let stream = StreamName::from_str("doe-stream-pairs").unwrap(); + let stream_id = seed_stream_with_meta( + &backend, + &basin, + &stream, + stream_meta_with_doe_min_age(MIN_AGE), + ) + .await; + + let write_timestamp = put_tail_position( + &backend, + stream_id, + StreamPosition { + seq_num: 1, + timestamp: 1234, + }, + ) + .await; + + backend + .process_stream_doe( + stream_id, + PendingDoeBatch::new(vec![ + kv::stream_doe_deadline::Entry { + deadline: deadline_after(write_timestamp, Duration::from_secs(50)), + min_age: Duration::from_secs(100), + }, + kv::stream_doe_deadline::Entry { + deadline: deadline_after(write_timestamp, Duration::from_secs(100)), + min_age: Duration::from_secs(10), + }, + ]), + ) + .await + .unwrap(); + + let meta = backend + .db + .get(kv::stream_meta::ser_key(&basin, &stream)) + .await + .unwrap() + .expect("stream meta should remain"); + let decoded = kv::stream_meta::deser_value(meta).unwrap(); + assert!(decoded.deleted_at.is_some()); + } + + #[tokio::test] + async fn reconfigure_enabling_doe_on_nonempty_retained_stream_arms_future_deadline() { + let backend = test_backend().await; + let basin = BasinName::from_str("doe-basin-enable").unwrap(); + let stream = StreamName::from_str("doe-stream-enable").unwrap(); + let config = OptionalStreamConfig { + retention_policy: Some(RetentionPolicy::Age(Duration::from_secs(120))), ..Default::default() }; - config.delete_on_empty.min_age = Some(Duration::from_secs(min_age)); - config - } - - async fn state( - backend: &Backend, - stream_id: StreamId, - ) -> Option<(kv::stream_doe_state::State, u64)> { - backend - .db_get_with(kv::stream_doe_state::ser_key(stream_id), |entry| { - Ok((kv::stream_doe_state::deser_value(entry.value)?, entry.seq)) - }) - .await - .unwrap() - } - - async fn scheduled(backend: &Backend, stream_id: StreamId) -> kv::stream_doe_state::Check { - let Some((kv::stream_doe_state::State::Scheduled(check), _)) = - state(backend, stream_id).await - else { - panic!("expected a scheduled check"); + let stream_id = seed_stream_with_meta( + &backend, + &basin, + &stream, + stream_meta_with_config(config, OffsetDateTime::now_utc()), + ) + .await; + let pos = StreamPosition { + seq_num: 1, + timestamp: 1234, }; - let checks = backend - .list_pending_stream_doe(TimestampSecs::MAX) - .await - .unwrap(); - assert_eq!(checks.values.len(), 1); - assert_eq!(checks.values[0].stream_id, stream_id); - assert_eq!(checks.values[0].check, check); - check - } - - async fn due(backend: &Backend, stream_id: StreamId) -> PendingCheck { - let txn = backend - .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - doe::schedule(&txn, stream_id, TimestampSecs::ZERO) - .await - .unwrap(); - db_txn_commit_durable(txn).await.unwrap(); - PendingCheck { - stream_id, - check: scheduled(backend, stream_id).await, - } - } - - async fn append(backend: &Backend, basin: &BasinName, stream: &StreamName, record: Record) { - let record: AppendRecord = AppendRecordParts { - timestamp: None, - record: record.metered(), - } - .try_into() - .unwrap(); backend - .open_for_append(basin, stream, None, OptionalStreamConfig::default()) - .await - .unwrap() - .append(AppendInput { - records: vec![record].try_into().unwrap(), - match_seq_num: None, - fencing_token: None, - }) + .db + .put( + kv::stream_tail_position::ser_key(stream_id), + kv::stream_tail_position::ser_value(pos), + ) + .await + .unwrap(); + backend + .db + .put( + kv::stream_record_timestamp::ser_key(stream_id, pos), + kv::stream_record_timestamp::ser_value(), + ) .await .unwrap(); - } - fn record() -> Record { - Record::try_from_parts(vec![], Bytes::from_static(b"live")).unwrap() - } - - async fn configure_min_age( - backend: &Backend, - basin: &BasinName, - stream: &StreamName, - min_age: u64, - via_ensure: bool, - ) { - let min_age = Duration::from_secs(min_age); - if via_ensure { - let mut config = backend - .get_stream_config(basin.clone(), stream.clone()) - .await - .unwrap(); - config.delete_on_empty.min_age = min_age; - backend - .provision_stream( - basin.clone(), - stream.clone(), - config.into(), - ProvisionMode::Ensure, - ) - .await - .unwrap(); - } else { - backend - .reconfigure_stream( - basin.clone(), - stream.clone(), - StreamReconfiguration { - delete_on_empty: Maybe::from(Some(DeleteOnEmptyReconfiguration { - min_age: Maybe::from(Some(min_age)), - })), - ..Default::default() - }, - ) - .await - .unwrap(); - } - } - - async fn configure_retention( - backend: &Backend, - basin: &BasinName, - stream: &StreamName, - age: u64, - ) { + let min_age = Duration::from_secs(30); + let expected_delay = + crate::backend::streamer::doe_arm_delay(Duration::from_secs(120), min_age); + let lower_bound = TimestampSecs::now(); backend .reconfigure_stream( - basin.clone(), - stream.clone(), + basin, + stream, StreamReconfiguration { - retention_policy: Maybe::from(Some(RetentionPolicy::Age(Duration::from_secs( - age, - )))), + delete_on_empty: Maybe::from(Some(DeleteOnEmptyReconfiguration { + min_age: Maybe::from(Some(min_age)), + })), ..Default::default() }, ) .await .unwrap(); - } + let upper_bound = TimestampSecs::now(); - async fn legacy(backend: &Backend, stream_id: StreamId, id: Option) -> Bytes { - let key = kv::stream_doe_deadline::ser_key(TimestampSecs::MAX, stream_id, id); - // Migration must not interpret the old min_age, even for future keys. - backend.db.put(&key, [0xff]).assert_durable().await; - key + let entries = list_doe_entries(&backend).await; + assert_eq!(entries.len(), 1); + let (deadline, scheduled_stream_id, scheduled_min_age) = entries[0]; + assert_eq!(scheduled_stream_id, stream_id); + assert_eq!(scheduled_min_age, min_age); + + let lower_secs = u64::from(lower_bound.as_u32()).saturating_add(expected_delay.as_secs()); + let upper_secs = u64::from(upper_bound.as_u32()).saturating_add(expected_delay.as_secs()); + let deadline_secs = u64::from(deadline.as_u32()); + assert!(lower_secs <= deadline_secs); + assert!(deadline_secs <= upper_secs); } #[tokio::test] - async fn creation_schedules_once_and_appends_do_not_write_schedules() { + async fn reconfigure_changing_enabled_doe_does_not_arm_new_deadline() { let backend = test_backend().await; - let before = TimestampSecs::after(Duration::from_secs(60)); - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - let check = scheduled(&backend, stream_id).await; - assert!(check.at >= before); - assert!(check.at <= TimestampSecs::after(Duration::from_secs(60))); - let original = state(&backend, stream_id).await; - append(&backend, &basin, &stream, record()).await; - assert_eq!(state(&backend, stream_id).await, original); - assert_eq!(scheduled(&backend, stream_id).await, check); - let mut old = backend - .db - .scan(kv::key_type_range(kv::KeyType::StreamDeleteOnEmptyDeadline)) - .await - .unwrap(); - assert!(old.next().await.unwrap().is_none()); - backend.close().await.unwrap(); - } - - #[rstest::rstest] - #[case::increase(3600, true)] - #[case::decrease(10, false)] - #[tokio::test] - async fn retry_uses_record_expiration_across_retention_changes( - #[case] new_age: u64, - #[case] append_after: bool, - ) { - let backend = test_backend().await; - let initial_age = if append_after { 10 } else { 3600 }; - let (basin, stream) = create_stream( - &backend, - config(60, RetentionPolicy::Age(Duration::from_secs(initial_age))), - ) - .await; - let stream_id = StreamId::new(&basin, &stream); - let lower_bound = TimestampSecs::after(Duration::from_secs(3600)); - append(&backend, &basin, &stream, record()).await; - let original = state(&backend, stream_id).await; - configure_retention(&backend, &basin, &stream, new_age).await; - if append_after { - append(&backend, &basin, &stream, record()).await; - } - assert_eq!(state(&backend, stream_id).await, original); - let pending = due(&backend, stream_id).await; - backend.process_stream_doe(pending).await.unwrap(); - let replacement = scheduled(&backend, stream_id).await; - assert_ne!(replacement.id, pending.check.id); - assert!( - replacement.at >= lower_bound, - "must preserve a check for the actual stored expiration" - ); - backend.close().await.unwrap(); - } - - #[rstest::rstest] - #[case::never_written(false)] - #[case::expired_records(true)] - #[tokio::test] - async fn empty_old_stream_is_deleted(#[case] written: bool) { - let backend = test_backend().await; - let (basin, stream) = create_stream( - &backend, - config(1, RetentionPolicy::Age(Duration::from_secs(1))), - ) - .await; - let stream_id = StreamId::new(&basin, &stream); - if written { - append(&backend, &basin, &stream, record()).await; - } - tokio::time::sleep(Duration::from_millis(1100)).await; - backend.clone().tick_stream_doe().await.unwrap(); - let meta = backend - .db_get( - kv::stream_meta::ser_key(&basin, &stream), - kv::stream_meta::deser_value, - ) - .await - .unwrap() - .unwrap(); - assert!(meta.deleted_at.is_some()); - assert!(state(&backend, stream_id).await.is_none()); - assert!( - backend - .list_pending_stream_doe(TimestampSecs::MAX) - .await - .unwrap() - .values - .is_empty() - ); - backend.clone().tick_stream_trim().await.unwrap(); - assert!( - backend - .db_get( - kv::stream_id_mapping::ser_key(stream_id), - kv::stream_id_mapping::deser_value - ) - .await - .unwrap() - .is_none() - ); - backend.close().await.unwrap(); - } - - #[rstest::rstest] - #[case::increase(3600)] - #[case::decrease(30)] - #[case::unchanged(60)] - #[case::disable(0)] - #[case::unrepresentable_cutoff(u64::MAX)] - #[tokio::test] - async fn configuration_coalesces_or_removes_the_schedule( - #[case] age: u64, - #[values(false, true)] via_ensure: bool, - ) { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - let original = state(&backend, stream_id).await; - configure_min_age(&backend, &basin, &stream, age, via_ensure).await; - if age == 0 { - assert!(state(&backend, stream_id).await.is_none()); - assert!( - backend - .list_pending_stream_doe(TimestampSecs::MAX) - .await - .unwrap() - .values - .is_empty() - ); - configure_min_age(&backend, &basin, &stream, 60, via_ensure).await; - assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); - } else if age == 60 { - assert_eq!(state(&backend, stream_id).await, original); - } else { - assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); - backend.clone().tick_stream_doe().await.unwrap(); - assert!( - scheduled(&backend, stream_id).await.at - >= TimestampSecs::now() - .saturating_add_duration(Duration::from_secs(age.saturating_sub(1))) - ); - assert!(backend.get_stream_config(basin, stream).await.is_ok()); - } - backend.close().await.unwrap(); - } - - #[tokio::test] - async fn partial_trim_wakes_parked_stream_with_finite_records_remaining() { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - append(&backend, &basin, &stream, record()).await; - backend - .process_stream_doe(due(&backend, stream_id).await) - .await - .unwrap(); - let parked = state(&backend, stream_id).await; - assert!(matches!( - parked, - Some((kv::stream_doe_state::State::Parked, _)) - )); - assert!( - backend - .list_pending_stream_doe(TimestampSecs::MAX) - .await - .unwrap() - .values - .is_empty() - ); - configure_retention(&backend, &basin, &stream, 3600).await; - append(&backend, &basin, &stream, record()).await; - assert_eq!(state(&backend, stream_id).await, parked); - append( + let basin = BasinName::from_str("doe-basin-stale").unwrap(); + let stream = StreamName::from_str("doe-stream-stale").unwrap(); + let initial_min_age = Duration::from_secs(10); + let mut config = OptionalStreamConfig::default(); + config.delete_on_empty.min_age = Some(initial_min_age); + let stream_id = seed_stream_with_meta( &backend, &basin, &stream, - Record::Command(CommandRecord::Trim(1)), + stream_meta_with_config(config, OffsetDateTime::now_utc()), ) .await; - backend.clone().tick_stream_trim().await.unwrap(); - assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); - backend.clone().tick_stream_doe().await.unwrap(); - assert!( - scheduled(&backend, stream_id).await.at - > TimestampSecs::after(Duration::from_secs(3500)) - ); - backend.close().await.unwrap(); - } - - #[rstest::rstest] - #[case::disabled(0, false)] - #[case::disabled_before_cleanup(60, false)] - #[case::legacy_only(60, true)] - #[tokio::test] - async fn trim_initializes_missing_state_only_for_enabled_empty_streams( - #[case] min_age: u64, - #[case] legacy_only: bool, - #[values(false, true)] full_trim: bool, - ) { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(min_age, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - append(&backend, &basin, &stream, record()).await; - append( - &backend, - &basin, - &stream, - Record::Command(CommandRecord::Trim(if full_trim { u64::MAX } else { 1 })), - ) - .await; - if legacy_only { - // Model an enabled stream that has not migrated yet. - let txn = backend - .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - doe::clear(&txn, stream_id).await.unwrap(); - db_txn_commit_durable(txn).await.unwrap(); - legacy(&backend, stream_id, None).await; - } else if min_age != 0 { - configure_min_age(&backend, &basin, &stream, 0, false).await; - } - assert!(state(&backend, stream_id).await.is_none()); - - backend.clone().tick_stream_trim().await.unwrap(); - let after_trim = state(&backend, stream_id).await; - if legacy_only && full_trim { - assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); - } else { - assert!(after_trim.is_none()); - assert!( - backend - .list_pending_stream_doe(TimestampSecs::MAX) - .await - .unwrap() - .values - .is_empty() - ); - } - - // Migration preserves a full trim's wake or initializes missing state. - if legacy_only { - backend.migrate_stream_doe().await.unwrap(); - if full_trim { - assert_eq!(state(&backend, stream_id).await, after_trim); - } - assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); - } - backend.close().await.unwrap(); - } - - #[tokio::test] - async fn full_trim_restores_doe_without_any_legacy_deadline() { - let backend = test_backend().await; - let (basin, stream) = create_stream(&backend, config(1, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - append(&backend, &basin, &stream, record()).await; - // Before upgrade, an infinite-retention stream normally loses its last - // deadline when it fires while the stream is nonempty. - let txn = backend + let existing_deadline = TimestampSecs::from_secs(4_242); + backend .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - doe::clear(&txn, stream_id).await.unwrap(); - db_txn_commit_durable(txn).await.unwrap(); - backend.migrate_stream_doe().await.unwrap(); - assert!(state(&backend, stream_id).await.is_none()); - - append( - &backend, - &basin, - &stream, - Record::Command(CommandRecord::Trim(u64::MAX)), - ) - .await; - backend.clone().tick_stream_trim().await.unwrap(); - assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); - tokio::time::sleep(Duration::from_millis(1100)).await; - backend.clone().tick_stream_doe().await.unwrap(); - let meta = backend - .db_get( - kv::stream_meta::ser_key(&basin, &stream), - kv::stream_meta::deser_value, + .put( + kv::stream_doe_deadline::ser_key(existing_deadline, stream_id), + kv::stream_doe_deadline::ser_value(initial_min_age), ) .await - .unwrap() .unwrap(); - assert!(meta.deleted_at.is_some()); - assert!(state(&backend, stream_id).await.is_none()); - backend.close().await.unwrap(); - } - #[rstest::rstest] - #[tokio::test] - async fn trim_wake_conflicts_with_concurrent_disablement( - #[values(false, true)] missing_state: bool, - ) { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - if missing_state { - let txn = backend - .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - doe::clear(&txn, stream_id).await.unwrap(); - db_txn_commit_durable(txn).await.unwrap(); - } - let txn = backend - .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - doe::wake_after_trim(&txn, stream_id, false).await.unwrap(); - configure_min_age(&backend, &basin, &stream, 0, false).await; - let error = StorageError::from(txn.commit().await.unwrap_err()); - assert!(error.is_transaction_conflict()); - assert!(state(&backend, stream_id).await.is_none()); - assert!( - backend - .list_pending_stream_doe(TimestampSecs::MAX) - .await - .unwrap() - .values - .is_empty() - ); - backend.close().await.unwrap(); - } - - #[rstest::rstest] - #[case::parked(TerminalTrimOutcome::Parked)] - #[case::deferred(TerminalTrimOutcome::RetryAt(TimestampSecs::MAX))] - #[tokio::test] - async fn event_retaining_same_ticket_invalidates_inflight_result( - #[values(false, true)] trim: bool, - #[case] outcome: TerminalTrimOutcome, - ) { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - append(&backend, &basin, &stream, record()).await; - if trim { - configure_retention(&backend, &basin, &stream, 3600).await; - } - let pending = due(&backend, stream_id).await; - let snapshot = backend.observe_doe_check(pending).await.unwrap().unwrap(); - if trim { - // The worker may already have observed the infinite-retention - // record and decided to park. Subsequent appends do not invalidate - // that observation, but removing the blocker must invalidate it. - append(&backend, &basin, &stream, record()).await; - append( - &backend, - &basin, - &stream, - Record::Command(CommandRecord::Trim(1)), + backend + .reconfigure_stream( + basin, + stream, + StreamReconfiguration { + delete_on_empty: Maybe::from(Some(DeleteOnEmptyReconfiguration { + min_age: Maybe::from(Some(Duration::from_secs(600))), + })), + ..Default::default() + }, ) - .await; - backend.clone().tick_stream_trim().await.unwrap(); - } else { - configure_min_age(&backend, &basin, &stream, 120, false).await; - } - assert_eq!(scheduled(&backend, stream_id).await, pending.check); - let woken = state(&backend, stream_id).await; - assert!(woken.unwrap().1 > snapshot.revision); - backend - .finish_doe_check(pending, snapshot, outcome) .await .unwrap(); - assert_eq!(state(&backend, stream_id).await, woken); - assert_eq!(scheduled(&backend, stream_id).await, pending.check); - backend.close().await.unwrap(); - } - #[tokio::test] - async fn obsolete_check_cleanup_preserves_wake_after_snapshot() { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - let pending = due(&backend, stream_id).await; - let observed = state(&backend, stream_id).await; - configure_min_age(&backend, &basin, &stream, 120, false).await; - let woken = state(&backend, stream_id).await; - assert!(woken.unwrap().1 > observed.unwrap().1); - assert_eq!(scheduled(&backend, stream_id).await, pending.check); - backend.discard_doe_check(pending, observed).await.unwrap(); - assert_eq!(state(&backend, stream_id).await, woken); - assert_eq!(scheduled(&backend, stream_id).await, pending.check); - backend.close().await.unwrap(); - } - - #[tokio::test] - async fn legacy_deadlines_only_initialize_missing_state() { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - let txn = backend - .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - doe::clear(&txn, stream_id).await.unwrap(); - db_txn_commit_durable(txn).await.unwrap(); - let old = legacy(&backend, stream_id, None).await; - let unique = legacy(&backend, stream_id, Some(42)).await; - backend.migrate_stream_doe().await.unwrap(); - assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); - assert!(backend.db.get(&old).await.unwrap().is_none()); - assert!(backend.db.get(&unique).await.unwrap().is_none()); - append(&backend, &basin, &stream, record()).await; - backend.clone().tick_stream_doe().await.unwrap(); - let parked = state(&backend, stream_id).await; - assert!(matches!( - parked, - Some((kv::stream_doe_state::State::Parked, _)) - )); - legacy(&backend, stream_id, Some(43)).await; - backend.migrate_stream_doe().await.unwrap(); - assert_eq!(state(&backend, stream_id).await, parked); - assert!( - backend - .list_pending_stream_doe(TimestampSecs::MAX) - .await - .unwrap() - .values - .is_empty() - ); - backend.close().await.unwrap(); - } - - #[tokio::test] - async fn migration_drains_pages_without_rewriting_schedules() { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - let before = state(&backend, stream_id).await; - let mut batch = slatedb::WriteBatch::new(); - for id in 0..=PENDING_LIST_LIMIT { - batch.put( - kv::stream_doe_deadline::ser_key(TimestampSecs::MAX, stream_id, Some(id as u128)), - [], - ); - } - backend.db.write(batch).assert_durable().await; - run_tick( - "stream-delete-on-empty", - &|backend: &Backend| backend.clone().tick_stream_doe(), - &backend, - ) - .await; - let mut remaining = backend - .db - .scan(kv::key_type_range(kv::KeyType::StreamDeleteOnEmptyDeadline)) - .await - .unwrap(); - assert!(remaining.next().await.unwrap().is_none()); - assert_eq!(state(&backend, stream_id).await, before); - scheduled(&backend, stream_id).await; - backend.close().await.unwrap(); - } - - #[tokio::test] - async fn migration_batches_cleanup_across_streams() { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let scheduled_id = StreamId::new(&basin, &stream); - let before = state(&backend, scheduled_id).await; - let disabled: StreamName = "disabled".parse().unwrap(); - let deleted: StreamName = "deleted".parse().unwrap(); - for (stream, min_age) in [(&disabled, 0), (&deleted, 60)] { - backend - .provision_stream( - basin.clone(), - stream.clone(), - config(min_age, RetentionPolicy::Infinite()), - ProvisionMode::Ensure, - ) - .await - .unwrap(); - } - backend - .delete_stream(basin.clone(), deleted.clone()) - .await - .unwrap(); - let disabled_id = StreamId::new(&basin, &disabled); - let deleted_id = StreamId::new(&basin, &deleted); - let missing_id = StreamId::new(&basin, &"missing".parse().unwrap()); - let deleted_state = state(&backend, deleted_id).await; - let mut keys = Vec::new(); - for stream_id in [scheduled_id, disabled_id, deleted_id, missing_id] { - for id in 0..3 { - keys.push(legacy(&backend, stream_id, Some(id)).await); - } - } - let before_seq = backend.db.snapshot().await.unwrap().seq(); - backend.migrate_stream_doe().await.unwrap(); + let entries = list_doe_entries(&backend).await; assert_eq!( - backend.db.snapshot().await.unwrap().seq(), - before_seq + 1, - "all cleanup-only streams should share one durable batch" + entries, + vec![(existing_deadline, stream_id, initial_min_age)] ); - for key in keys { - assert!(backend.db.get(key).await.unwrap().is_none()); - } - assert_eq!(state(&backend, scheduled_id).await, before); - assert_eq!(state(&backend, deleted_id).await, deleted_state); - assert!(state(&backend, disabled_id).await.is_none()); - assert!(state(&backend, missing_id).await.is_none()); - backend.close().await.unwrap(); - } - - #[rstest::rstest] - #[tokio::test] - async fn migration_revalidates_configuration_after_snapshot(#[values(0, 120)] min_age: u64) { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - let txn = backend - .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - doe::clear(&txn, stream_id).await.unwrap(); - db_txn_commit_durable(txn).await.unwrap(); - let key = legacy(&backend, stream_id, None).await; - let snapshot = backend.db.snapshot().await.unwrap(); - configure_min_age(&backend, &basin, &stream, min_age, false).await; - let configured = state(&backend, stream_id).await; - backend - .migrate_doe_deadlines(&snapshot, stream_id, vec![key.clone()]) - .await - .unwrap(); - assert_eq!(state(&backend, stream_id).await, configured); - assert!(backend.db.get(key).await.unwrap().is_none()); - backend.close().await.unwrap(); - } - - #[tokio::test] - async fn concurrent_wake_conflicts_with_completion_transaction() { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - let pending = due(&backend, stream_id).await; - let txn = backend - .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - let previous = doe::state(&txn, stream_id).await.unwrap(); - doe::replace( - &txn, - stream_id, - previous, - Some(kv::stream_doe_state::State::Parked), - ) - .unwrap(); - // The same ticket is retained, but the scheduler revision changes. - let wake = backend - .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - doe::schedule(&wake, stream_id, TimestampSecs::now()) - .await - .unwrap(); - db_txn_commit_durable(wake).await.unwrap(); - let error = txn.commit().await.unwrap_err(); - assert_eq!(error.kind(), slatedb::ErrorKind::Transaction); - assert_eq!(scheduled(&backend, stream_id).await, pending.check); - backend.close().await.unwrap(); - } - - #[tokio::test] - async fn full_trim_wakes_parked_stream_and_waits_for_minimum_age() { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - append(&backend, &basin, &stream, record()).await; - backend - .process_stream_doe(due(&backend, stream_id).await) - .await - .unwrap(); - assert!(matches!( - state(&backend, stream_id).await, - Some((kv::stream_doe_state::State::Parked, _)) - )); - append( - &backend, - &basin, - &stream, - Record::Command(CommandRecord::Trim(u64::MAX)), - ) - .await; - backend.clone().tick_stream_trim().await.unwrap(); - assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); - backend.clone().tick_stream_doe().await.unwrap(); - assert!(scheduled(&backend, stream_id).await.at > TimestampSecs::now()); - let meta = backend - .db_get( - kv::stream_meta::ser_key(&basin, &stream), - kv::stream_meta::deser_value, - ) - .await - .unwrap() - .unwrap(); - assert!(meta.deleted_at.is_none()); - backend.close().await.unwrap(); - } - - #[rstest::rstest] - #[case::deletion_pending(TerminalTrimOutcome::DeletionPending)] - #[case::parked(TerminalTrimOutcome::Parked)] - #[tokio::test] - async fn stale_work_cannot_delete_or_reschedule_recreated_stream( - #[case] outcome: TerminalTrimOutcome, - ) { - let backend = test_backend().await; - let configuration = config(60, RetentionPolicy::Infinite()); - let (basin, stream) = create_stream(&backend, configuration.clone()).await; - let stream_id = StreamId::new(&basin, &stream); - let pending = due(&backend, stream_id).await; - let snapshot = backend.observe_doe_check(pending).await.unwrap().unwrap(); - backend - .delete_stream(basin.clone(), stream.clone()) - .await - .unwrap(); - backend.clone().tick_stream_trim().await.unwrap(); - backend - .provision_stream( - basin.clone(), - stream.clone(), - configuration, - ProvisionMode::Ensure, - ) - .await - .unwrap(); - let recreated = state(&backend, stream_id).await; - let client = backend - .streamer_client_guarded(&basin, &stream) - .await - .unwrap(); - assert_eq!( - client - .terminal_trim(TerminalTrimCondition::DeleteOnEmpty { - expected_stream_creation_seq: snapshot.creation_seq, - expected_config_seq: snapshot.config_seq, - }) - .await - .unwrap(), - TerminalTrimOutcome::Obsolete - ); - backend - .finish_doe_check(pending, snapshot, outcome) - .await - .unwrap(); - backend.process_stream_doe(pending).await.unwrap(); - legacy(&backend, stream_id, None).await; - backend.migrate_stream_doe().await.unwrap(); - assert_eq!(state(&backend, stream_id).await, recreated); - scheduled(&backend, stream_id).await; - backend.close().await.unwrap(); } } diff --git a/lite/src/backend/bgtasks/stream_trim.rs b/lite/src/backend/bgtasks/stream_trim.rs index db377d0c..2a1f332d 100644 --- a/lite/src/backend/bgtasks/stream_trim.rs +++ b/lite/src/backend/bgtasks/stream_trim.rs @@ -3,19 +3,13 @@ use std::ops::RangeTo; use futures::{StreamExt, stream}; use s2_common::{record::NonZeroSeqNum, resources::Page}; use slatedb::{ - IsolationLevel, WriteBatch, + WriteBatch, config::{DurabilityLevel, ScanOptions}, }; use tracing::instrument; -use super::PageProgress; use crate::{ - backend::{ - Backend, doe, - error::StorageError, - kv, - store::{db_txn_commit_durable, db_txn_get, db_txn_get_with}, - }, + backend::{Backend, error::StorageError, kv, store::db_txn_get}, stream_id::StreamId, }; @@ -23,34 +17,27 @@ const PENDING_LIST_LIMIT: usize = 128; const CONCURRENCY: usize = 4; const DELETE_BATCH_SIZE: usize = 10_000; -#[derive(Debug, Clone, Copy)] -struct PendingTrim { - stream_id: StreamId, - trim_point: RangeTo, - /// Commit sequence of the observed marker; bounds the records this job may delete. - marker_seq: u64, -} - impl Backend { pub(super) async fn tick_stream_trim(self) -> Result { let page = self.list_stream_trim_pending().await?; if page.values.is_empty() { return Ok(page.has_more); } - let mut progress = PageProgress::new(page.has_more); let mut processed = stream::iter(page.values) - .map(|pending| { + .map(|(stream_id, trim_point)| { let backend = self.clone(); - async move { backend.process_trim(pending).await } + async move { backend.process_trim(stream_id, trim_point).await } }) .buffer_unordered(CONCURRENCY); while let Some(result) = processed.next().await { - progress.record_result(result, StorageError::is_transaction_conflict)?; + result?; } - Ok(progress.should_continue()) + Ok(page.has_more) } - async fn list_stream_trim_pending(&self) -> Result, StorageError> { + async fn list_stream_trim_pending( + &self, + ) -> Result)>, StorageError> { let scan_opts = ScanOptions { durability_filter: DurabilityLevel::Remote, ..Default::default() @@ -63,11 +50,7 @@ impl Backend { while let Some(kv) = it.next().await? { let stream_id = kv::stream_trim_point::deser_key(kv.key)?; let trim_point = kv::stream_trim_point::deser_value(kv.value)?; - pending.push(PendingTrim { - stream_id, - trim_point, - marker_seq: kv.seq, - }); + pending.push((stream_id, trim_point)); if pending.len() >= PENDING_LIST_LIMIT { return Ok(Page::new(pending, true)); } @@ -75,15 +58,26 @@ impl Backend { Ok(Page::new(pending, false)) } - async fn process_trim(&self, pending: PendingTrim) -> Result<(), StorageError> { - let has_remaining_records = self.delete_records(pending).await?; - self.finalize_trim(pending, has_remaining_records).await + async fn process_trim( + &self, + stream_id: StreamId, + trim_point: RangeTo, + ) -> Result<(), StorageError> { + let has_remaining_records = self.delete_records(stream_id, trim_point).await?; + if trim_point.end < NonZeroSeqNum::MAX && !has_remaining_records { + self.arm_doe_on_full_trim(stream_id).await?; + } + self.finalize_trim(stream_id, trim_point).await?; + Ok(()) } - /// Return whether records remain beyond the trim point. #[instrument(ret, err, skip(self))] - async fn delete_records(&self, pending: PendingTrim) -> Result { - let prefix = kv::stream_record_timestamp::ser_key_prefix(pending.stream_id); + async fn delete_records( + &self, + stream_id: StreamId, + trim_point: RangeTo, + ) -> Result { + let prefix = kv::stream_record_timestamp::ser_key_prefix(stream_id); let scan_opts = ScanOptions { durability_filter: DurabilityLevel::Remote, ..Default::default() @@ -93,69 +87,67 @@ impl Backend { .scan_prefix_with_options(prefix, .., &scan_opts) .await?; let mut batch = WriteBatch::new(); - let mut batch_size = 0; + let mut batch_size = 0usize; let mut has_remaining_records = false; while let Some(kv) = it.next().await? { let (deser_stream_id, pos) = kv::stream_record_timestamp::deser_key(kv.key.clone())?; - debug_assert_eq!(deser_stream_id, pending.stream_id); - // Name reuse requires all old records to be durably deleted first. - // A stale job may see records from the recreated stream; stop before - // deleting records committed after the trim marker it observed. - if pos.seq_num >= pending.trim_point.end.get() || kv.seq > pending.marker_seq { + debug_assert_eq!(deser_stream_id, stream_id); + if pos.seq_num >= trim_point.end.get() { has_remaining_records = true; break; } batch.delete(kv.key); - batch.delete(kv::stream_record_data::ser_key(pending.stream_id, pos)); + batch.delete(kv::stream_record_data::ser_key(stream_id, pos)); batch_size += 1; if batch_size >= DELETE_BATCH_SIZE { - self.db.write(batch).await?.await_durable().await?; + self.db.write(batch).await?; batch = WriteBatch::new(); batch_size = 0; } } - if !batch.is_empty() { - self.db.write(batch).await?.await_durable().await?; + if batch_size > 0 { + self.db.write(batch).await?; } Ok(has_remaining_records) } - #[instrument(skip(self))] + #[instrument(ret, err, skip(self))] async fn finalize_trim( &self, - pending: PendingTrim, - has_remaining_records: bool, + stream_id: StreamId, + trim_point: RangeTo, ) -> Result<(), StorageError> { - let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; - let trim_point_key = kv::stream_trim_point::ser_key(pending.stream_id); - let current_seq = db_txn_get_with(&txn, &trim_point_key, |entry| Ok(entry.seq)).await?; - // A tick can exit on error while its submitted cleanup is still running. - // The next durable scan can capture the old marker before cleanup becomes - // durable. Cleanup removes the old marker and metadata before name reuse, - // but this scanned work can remain queued across recreation. The recreated - // stream's marker may have the same trim point, so compare commit sequences. - if current_seq != Some(pending.marker_seq) { + let trim_point_key = kv::stream_trim_point::ser_key(stream_id); + let txn = self + .db + .begin(slatedb::IsolationLevel::SerializableSnapshot) + .await?; + let is_full_delete = trim_point == ..NonZeroSeqNum::MAX; + let Some(current_trim_point) = db_txn_get( + &txn, + trim_point_key.clone(), + kv::stream_trim_point::deser_value, + ) + .await? + else { + return Ok(()); + }; + if current_trim_point != trim_point { return Ok(()); } - let is_terminal_trim = pending.trim_point == ..NonZeroSeqNum::MAX; txn.delete(trim_point_key)?; - if is_terminal_trim { - let id_mapping_key = kv::stream_id_mapping::ser_key(pending.stream_id); + if is_full_delete { + let id_mapping_key = kv::stream_id_mapping::ser_key(stream_id); if let Some((basin, stream)) = db_txn_get(&txn, &id_mapping_key, kv::stream_id_mapping::deser_value).await? { txn.delete(kv::stream_meta::ser_key(&basin, &stream))?; txn.delete(id_mapping_key)?; } - txn.delete(kv::stream_tail_position::ser_key(pending.stream_id))?; - txn.delete(kv::stream_fencing_token::ser_key(pending.stream_id))?; - doe::clear(&txn, pending.stream_id).await?; - } else { - // A partial trim may remove the infinite-retention record that - // parked DOE while leaving finite-retention records behind. - doe::wake_after_trim(&txn, pending.stream_id, has_remaining_records).await?; + txn.delete(kv::stream_tail_position::ser_key(stream_id))?; + txn.delete(kv::stream_fencing_token::ser_key(stream_id))?; } - db_txn_commit_durable(txn).await?; + txn.commit().await?; Ok(()) } } @@ -178,10 +170,7 @@ mod tests { use time::OffsetDateTime; use super::super::tests::test_backend; - use crate::{ - backend::{kv, test_util::DbWriteTestExt as _}, - stream_id::StreamId, - }; + use crate::{backend::kv, stream_id::StreamId}; fn test_record() -> Metered { let record = Record::try_from_parts(vec![], Bytes::from_static(b"trim-test")).unwrap(); @@ -209,16 +198,16 @@ mod tests { kv::stream_record_data::ser_key(stream_id, pos), kv::stream_record_data::ser_value(metered.as_ref()), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( kv::stream_record_timestamp::ser_key(stream_id, pos), kv::stream_record_timestamp::ser_value(), ) - .assert_durable() - .await; + .await + .unwrap(); } backend @@ -227,8 +216,8 @@ mod tests { kv::stream_trim_point::ser_key(stream_id), kv::stream_trim_point::ser_value(trim_point(3)), ) - .assert_durable() - .await; + .await + .unwrap(); backend.clone().tick_stream_trim().await.unwrap(); @@ -286,16 +275,16 @@ mod tests { kv::stream_meta::ser_key(&basin, &stream), kv::stream_meta::ser_value(&meta), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( kv::stream_id_mapping::ser_key(stream_id), kv::stream_id_mapping::ser_value(&basin, &stream), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( @@ -305,8 +294,8 @@ mod tests { timestamp: 1234, }), ) - .assert_durable() - .await; + .await + .unwrap(); let token = FencingToken::from_str("token-1").unwrap(); backend .db @@ -314,8 +303,8 @@ mod tests { kv::stream_fencing_token::ser_key(stream_id), kv::stream_fencing_token::ser_value(&token), ) - .assert_durable() - .await; + .await + .unwrap(); for seq in 0..3 { let pos = StreamPosition { @@ -328,16 +317,16 @@ mod tests { kv::stream_record_data::ser_key(stream_id, pos), kv::stream_record_data::ser_value(metered.as_ref()), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( kv::stream_record_timestamp::ser_key(stream_id, pos), kv::stream_record_timestamp::ser_value(), ) - .assert_durable() - .await; + .await + .unwrap(); } backend @@ -346,8 +335,8 @@ mod tests { kv::stream_trim_point::ser_key(stream_id), kv::stream_trim_point::ser_value(trim_point(SeqNum::MAX)), ) - .assert_durable() - .await; + .await + .unwrap(); backend.clone().tick_stream_trim().await.unwrap(); @@ -402,6 +391,35 @@ mod tests { } } + #[tokio::test] + async fn stream_trim_skips_stale_trim_point() { + let backend = test_backend().await; + let stream_id: StreamId = [9u8; StreamId::LEN].into(); + + backend + .db + .put( + kv::stream_trim_point::ser_key(stream_id), + kv::stream_trim_point::ser_value(trim_point(10)), + ) + .await + .unwrap(); + + backend + .finalize_trim(stream_id, trim_point(5)) + .await + .unwrap(); + + let current = backend + .db + .get(kv::stream_trim_point::ser_key(stream_id)) + .await + .unwrap() + .expect("trim point should remain"); + let decoded = kv::stream_trim_point::deser_value(current).unwrap(); + assert_eq!(decoded, trim_point(10)); + } + #[tokio::test] async fn stream_trim_paginates_pending_list() { let backend = test_backend().await; @@ -417,7 +435,7 @@ mod tests { kv::stream_trim_point::ser_value(trim_point(1)), ); } - backend.db.write(batch).assert_durable().await; + backend.db.write(batch).await.unwrap(); let has_more = backend.clone().tick_stream_trim().await.unwrap(); assert!(has_more); @@ -454,24 +472,24 @@ mod tests { kv::stream_record_data::ser_key(stream_id, pos), kv::stream_record_data::ser_value(metered.as_ref()), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( kv::stream_record_timestamp::ser_key(stream_id, pos), kv::stream_record_timestamp::ser_value(), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( kv::stream_trim_point::ser_key(stream_id), kv::stream_trim_point::ser_value(trim_point(1)), ) - .assert_durable() - .await; + .await + .unwrap(); backend.clone().tick_stream_trim().await.unwrap(); @@ -496,6 +514,110 @@ mod tests { assert!(trim_point.is_none()); } + #[tokio::test] + async fn stream_trim_does_not_touch_other_streams() { + let backend = test_backend().await; + let stream_id_a: StreamId = [1u8; StreamId::LEN].into(); + let stream_id_b: StreamId = [2u8; StreamId::LEN].into(); + let metered = test_record(); + + for seq in 0..4 { + let pos_a = StreamPosition { + seq_num: seq, + timestamp: 1000 + seq, + }; + backend + .db + .put( + kv::stream_record_data::ser_key(stream_id_a, pos_a), + kv::stream_record_data::ser_value(metered.as_ref()), + ) + .await + .unwrap(); + backend + .db + .put( + kv::stream_record_timestamp::ser_key(stream_id_a, pos_a), + kv::stream_record_timestamp::ser_value(), + ) + .await + .unwrap(); + + let pos_b = StreamPosition { + seq_num: seq, + timestamp: 2000 + seq, + }; + backend + .db + .put( + kv::stream_record_data::ser_key(stream_id_b, pos_b), + kv::stream_record_data::ser_value(metered.as_ref()), + ) + .await + .unwrap(); + backend + .db + .put( + kv::stream_record_timestamp::ser_key(stream_id_b, pos_b), + kv::stream_record_timestamp::ser_value(), + ) + .await + .unwrap(); + } + + backend + .db + .put( + kv::stream_trim_point::ser_key(stream_id_a), + kv::stream_trim_point::ser_value(trim_point(2)), + ) + .await + .unwrap(); + + backend.clone().tick_stream_trim().await.unwrap(); + + for seq in 0..4 { + let pos_a = StreamPosition { + seq_num: seq, + timestamp: 1000 + seq, + }; + let data_a = backend + .db + .get(kv::stream_record_data::ser_key(stream_id_a, pos_a)) + .await + .unwrap(); + let timestamp_a = backend + .db + .get(kv::stream_record_timestamp::ser_key(stream_id_a, pos_a)) + .await + .unwrap(); + if seq < 2 { + assert!(data_a.is_none()); + assert!(timestamp_a.is_none()); + } else { + assert!(data_a.is_some()); + assert!(timestamp_a.is_some()); + } + + let pos_b = StreamPosition { + seq_num: seq, + timestamp: 2000 + seq, + }; + let data_b = backend + .db + .get(kv::stream_record_data::ser_key(stream_id_b, pos_b)) + .await + .unwrap(); + let timestamp_b = backend + .db + .get(kv::stream_record_timestamp::ser_key(stream_id_b, pos_b)) + .await + .unwrap(); + assert!(data_b.is_some()); + assert!(timestamp_b.is_some()); + } + } + #[tokio::test] async fn stream_trim_large_batch_flushes() { let backend = test_backend().await; @@ -523,7 +645,7 @@ mod tests { kv::stream_trim_point::ser_key(stream_id), kv::stream_trim_point::ser_value(trim_point(total)), ); - backend.db.write(batch).assert_durable().await; + backend.db.write(batch).await.unwrap(); backend.clone().tick_stream_trim().await.unwrap(); @@ -556,149 +678,47 @@ mod tests { } #[tokio::test] - async fn stale_deletion_work_preserves_recreated_stream() { - use s2_common::{ - config::OptionalStreamConfig, - resources::ProvisionMode, - stream::{AppendInput, AppendRecord, AppendRecordParts}, - }; - - use crate::backend::error::{ProvisionStreamError, ReconfigureStreamError}; + async fn finalize_trim_no_trim_point_noop() { let backend = test_backend().await; - let (basin, stream) = - crate::backend::test_util::create_stream(&backend, OptionalStreamConfig::default()) - .await; - let stream_id = StreamId::new(&basin, &stream); + let stream_id: StreamId = [4u8; StreamId::LEN].into(); + backend - .open_for_check_tail(&basin, &stream) - .await - .unwrap() - .check_tail() + .finalize_trim(stream_id, trim_point(5)) .await .unwrap(); - // Pause stream deletion after it submits terminal trim, before it marks metadata. - let mut deletion = Box::pin(backend.delete_stream(basin.clone(), stream.clone())); - assert!(futures::poll!(&mut deletion).is_pending()); - tokio::time::timeout(std::time::Duration::from_secs(5), async { - loop { - if backend - .db_get( - kv::stream_trim_point::ser_key(stream_id), - kv::stream_trim_point::deser_value, - ) - .await - .unwrap() - == Some(..NonZeroSeqNum::MAX) - { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - // A crash here leaves a durable terminal trim with unmarked metadata. - // Provisioning and reconfiguration must reject updates that the pending trim would erase. - assert!(matches!( - backend - .provision_stream( - basin.clone(), - stream.clone(), - OptionalStreamConfig::default(), - ProvisionMode::Ensure - ) - .await, - Err(ProvisionStreamError::StreamDeletionPending(_)) - )); - assert!(matches!( - backend - .reconfigure_stream(basin.clone(), stream.clone(), Default::default()) - .await, - Err(ReconfigureStreamError::StreamDeletionPending(_)) - )); - - let stale_trim = backend - .list_stream_trim_pending() + let trim_point = backend + .db + .get(kv::stream_trim_point::ser_key(stream_id)) .await - .unwrap() - .values - .pop() .unwrap(); - backend.clone().tick_stream_trim().await.unwrap(); + assert!(trim_point.is_none()); + } + + #[tokio::test] + async fn finalize_trim_clears_matching_trim_point() { + let backend = test_backend().await; + let stream_id: StreamId = [5u8; StreamId::LEN].into(); + backend - .provision_stream( - basin.clone(), - stream.clone(), - OptionalStreamConfig::default(), - ProvisionMode::CreateOnly { - request_token: None, - }, + .db + .put( + kv::stream_trim_point::ser_key(stream_id), + kv::stream_trim_point::ser_value(trim_point(5)), ) .await .unwrap(); - deletion.await.unwrap(); - let meta = backend - .db_get( - kv::stream_meta::ser_key(&basin, &stream), - kv::stream_meta::deser_value, - ) - .await - .unwrap() - .unwrap(); - assert!( - meta.deleted_at.is_none(), - "the old deletion request must not mark the recreated stream as deleted" - ); - - let record: AppendRecord = AppendRecordParts { - timestamp: None, - record: Record::try_from_parts(vec![], Bytes::from_static(b"recreated stream")) - .unwrap() - .metered(), - } - .try_into() - .unwrap(); - let ack = backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) - .await - .unwrap() - .append(AppendInput { - records: vec![record].try_into().unwrap(), - match_seq_num: None, - fencing_token: None, - }) + backend + .finalize_trim(stream_id, trim_point(5)) .await .unwrap(); - // A previous tick can have scanned the old marker before cleanup became durable. - backend.process_trim(stale_trim).await.unwrap(); - assert!( - backend - .db_get( - kv::stream_record_data::ser_key(stream_id, ack.start), - kv::stream_record_data::deser_value, - ) - .await - .unwrap() - .is_some(), - "old trim work must not delete records from the recreated stream" - ); - // Even another terminal marker belongs to different work after recreation. - backend.delete_stream(basin, stream).await.unwrap(); - backend.process_trim(stale_trim).await.unwrap(); - assert_eq!( - backend - .db_get( - kv::stream_trim_point::ser_key(stream_id), - kv::stream_trim_point::deser_value, - ) - .await - .unwrap(), - Some(..NonZeroSeqNum::MAX), - "old trim work must not finalize the recreated stream's deletion" - ); - backend.close().await.unwrap(); + let trim_point = backend + .db + .get(kv::stream_trim_point::ser_key(stream_id)) + .await + .unwrap(); + assert!(trim_point.is_none()); } } diff --git a/lite/src/backend/core.rs b/lite/src/backend/core.rs index 25359c60..0b915877 100644 --- a/lite/src/backend/core.rs +++ b/lite/src/backend/core.rs @@ -8,12 +8,13 @@ use futures::{ }; use s2_common::{ basin::BasinName, - config::{BasinConfig, OptionalStreamConfig, StreamConfig}, + config::{BasinConfig, OptionalStreamConfig}, encryption::{EncryptionAlgorithm, EncryptionSpec}, record::{NonZeroSeqNum, SeqNum, StreamPosition}, resources::ProvisionMode, stream::StreamName, }; +use slatedb::config::{DurabilityLevel, ReadOptions, ScanOptions}; use tokio::sync::{Semaphore, broadcast}; use super::{ @@ -25,9 +26,7 @@ use super::{ StreamerMissingInActionError, TransactionConflictError, }, kv, - store::db_snapshot_get_with, streamer::{GuardedStreamerClient, StreamerClient, StreamerGenerationId}, - timestamp::TimestampSecs, }; use crate::{backend::bgtasks::BgtaskTrigger, stream_id::StreamId}; @@ -38,7 +37,6 @@ enum StreamerClientSlot { Initializing { generation_id: StreamerGenerationId, future: StreamerInitFuture, - pending_config: Option<(u64, StreamConfig)>, }, Ready { client: StreamerClient, @@ -114,74 +112,46 @@ impl Backend { basin: BasinName, stream: StreamName, ) -> Result { - // Read one consistent, durable view of the stream. - let snapshot = self.db.snapshot().await.map_err(StorageError::from)?; - self.await_durable_seq(snapshot.seq()).await?; let stream_id = StreamId::new(&basin, &stream); - let (meta, persisted_tail, fencing_token, trim_point, stream_creation_seq) = tokio::try_join!( - db_snapshot_get_with( - &snapshot, + let (meta, persisted_tail, fencing_token, trim_point) = tokio::try_join!( + self.db_get( kv::stream_meta::ser_key(&basin, &stream), - |entry| { Ok((kv::stream_meta::deser_value(entry.value)?, entry.seq)) } + kv::stream_meta::deser_value, ), - db_snapshot_get_with( - &snapshot, - kv::stream_tail_position::ser_key(stream_id), - |entry| { - Ok(( - kv::stream_tail_position::deser_value(entry.value)?, - TimestampSecs::from_millis(entry.create_ts), - )) - } - ), - db_snapshot_get_with( - &snapshot, + self.load_persisted_stream_tail(stream_id), + self.db_get( kv::stream_fencing_token::ser_key(stream_id), - |entry| kv::stream_fencing_token::deser_value(entry.value), + kv::stream_fencing_token::deser_value, ), - db_snapshot_get_with( - &snapshot, + self.db_get( kv::stream_trim_point::ser_key(stream_id), - |entry| kv::stream_trim_point::deser_value(entry.value), - ), - db_snapshot_get_with( - &snapshot, - kv::stream_id_mapping::ser_key(stream_id), - |entry| Ok(entry.seq) - ), + kv::stream_trim_point::deser_value, + ) )?; - let Some((meta, config_seq)) = meta else { + let Some(meta) = meta else { return Err(StreamNotFoundError { basin, stream }.into()); }; let (tail_pos, last_tail_write_timestamp) = - persisted_tail.unwrap_or((StreamPosition::MIN, TimestampSecs::ZERO)); + persisted_tail.unwrap_or((StreamPosition::MIN, kv::timestamp::TimestampSecs::ZERO)); - if meta.deleted_at.is_some() || trim_point == Some(..NonZeroSeqNum::MAX) { - return Err(StreamDeletionPendingError.into()); - } - - self.assert_no_records_following_tail(&snapshot, stream_id, &basin, &stream, tail_pos) + self.assert_no_records_following_tail(stream_id, &basin, &stream, tail_pos) .await?; let fencing_token = fencing_token.unwrap_or_default(); - let stream_creation_seq = stream_creation_seq.ok_or_else(|| { - StorageError::InvariantViolation(format!( - "live stream `{basin}/{stream}` has no ID mapping" - )) - })?; + if trim_point == Some(..NonZeroSeqNum::MAX) { + return Err(StreamDeletionPendingError.into()); + } let streamer_slots = self.streamer_slots.clone(); Ok(super::streamer::Spawner { generation_id, db: self.db.clone(), stream_id, - stream_creation_seq, config: meta.config, - config_seq, cipher: meta.cipher, tail_pos, last_tail_write_timestamp, @@ -198,9 +168,29 @@ impl Backend { })) } + async fn load_persisted_stream_tail( + &self, + stream_id: StreamId, + ) -> Result, StorageError> { + let read_opts = ReadOptions { + durability_filter: DurabilityLevel::Remote, + ..Default::default() + }; + let Some(entry) = self + .db + .get_key_value_with_options(kv::stream_tail_position::ser_key(stream_id), &read_opts) + .await? + else { + return Ok(None); + }; + Ok(Some(( + kv::stream_tail_position::deser_value(entry.value)?, + kv::timestamp::TimestampSecs::from_millis(entry.create_ts), + ))) + } + async fn assert_no_records_following_tail( &self, - snapshot: &slatedb::DbSnapshot, stream_id: StreamId, basin: &BasinName, stream: &StreamName, @@ -211,7 +201,14 @@ impl Backend { seq_num: tail_pos.seq_num, timestamp: 0, }); - let mut it = snapshot.scan_prefix(prefix, start_suffix..).await?; + let scan_opts = ScanOptions { + durability_filter: DurabilityLevel::Remote, + ..Default::default() + }; + let mut it = self + .db + .scan_prefix_with_options(prefix, start_suffix.., &scan_opts) + .await?; let Some(kv) = it.next().await? else { return Ok(()); }; @@ -244,22 +241,13 @@ impl Backend { fn new_initializing_slot(self, basin: &BasinName, stream: &StreamName) -> StreamerClientSlot { let basin = basin.clone(); let stream = stream.clone(); - let stream_id = StreamId::new(&basin, &stream); let generation_id = StreamerGenerationId::next(); - // Drive initialization independently so cancelled callers cannot leave - // its snapshot pinned in a cached, unpolled future. - let future = tokio::spawn(async move { - let result = self.start_streamer(generation_id, basin, stream).await; - self.streamer_finish_initialization(stream_id, generation_id, &result); - result - }) - .map(|result| result.expect("streamer initialization task panicked")) - .boxed() - .shared(); + let future = async move { self.start_streamer(generation_id, basin, stream).await } + .boxed() + .shared(); StreamerClientSlot::Initializing { generation_id, future, - pending_config: None, } } @@ -284,13 +272,6 @@ impl Backend { if client.is_dead() { oe.remove(); } else { - if let StreamerClientSlot::Initializing { - pending_config: Some((seq, config)), - .. - } = oe.get() - { - client.advise_reconfig(*seq, config.clone()); - } oe.insert(StreamerClientSlot::Ready { client: client.clone(), }); @@ -309,35 +290,30 @@ impl Backend { basin: &BasinName, stream: &StreamName, ) -> Result { + let stream_id = StreamId::new(basin, stream); match self.streamer_client_slot(basin, stream) { - StreamerClientSlot::Initializing { future, .. } => future.await, + StreamerClientSlot::Initializing { + generation_id, + future, + } => { + let result = future.await; + self.streamer_finish_initialization(stream_id, generation_id, &result); + result + } StreamerClientSlot::Ready { client } => Ok(client), } } - pub(super) fn advise_stream_config( + pub(super) fn streamer_client_if_active( &self, basin: &BasinName, stream: &StreamName, - seq: u64, - config: StreamConfig, - ) { + ) -> Option { let stream_id = StreamId::new(basin, stream); - let Some(mut slot) = self.streamer_slots.get_mut(&stream_id) else { - return; - }; - match slot.value_mut() { - StreamerClientSlot::Ready { client } => { - client.advise_reconfig(seq, config); - } - StreamerClientSlot::Initializing { pending_config, .. } => { - if pending_config - .as_ref() - .is_none_or(|(pending_seq, _)| seq > *pending_seq) - { - *pending_config = Some((seq, config)); - } - } + let slot = self.streamer_slots.get(&stream_id)?; + match slot.value() { + StreamerClientSlot::Ready { client } if !client.is_dead() => Some(client.clone()), + _ => None, } } @@ -355,16 +331,11 @@ impl Backend { } } - /// Resolve a handle for `stream`, creating it on demand if the basin config allows. - /// - /// `stream_config` is applied over the basin's default stream configuration only if the - /// stream is being created. It must already be validated. pub(super) async fn stream_handle_with_auto_create( &self, basin: &BasinName, stream: &StreamName, - auto_create_on: AutoCreateOn, - stream_config: OptionalStreamConfig, + should_auto_create: impl FnOnce(&BasinConfig) -> bool, resolve_encryption: impl FnOnce(Option) -> Result, ) -> Result where @@ -376,66 +347,54 @@ impl Backend { + From + From, { - let client = match self.streamer_client_guarded(basin, stream).await { - Ok(client) => client, + match self.streamer_client_guarded(basin, stream).await { + Ok(client) => Ok(StreamHandle { + db: self.db.clone(), + encryption: resolve_encryption(client.cipher())?, + client, + }), Err(StreamerError::StreamNotFound(e)) => { let config = match self.get_basin_config(basin.clone()).await { Ok(config) => config, Err(GetBasinConfigError::Storage(e)) => Err(e)?, Err(GetBasinConfigError::BasinNotFound(e)) => Err(e)?, }; - if !auto_create_on.is_enabled(&config) { - return Err(e.into()); - } - if let Err(e) = self - .provision_stream( - basin.clone(), - stream.clone(), - stream_config, - ProvisionMode::CreateOnly { - request_token: None, - }, - ) - .await - { - match e { - ProvisionStreamError::Storage(e) => Err(e)?, - ProvisionStreamError::TransactionConflict(e) => Err(e)?, - ProvisionStreamError::BasinDeletionPending(e) => Err(e)?, - ProvisionStreamError::StreamDeletionPending(e) => Err(e)?, - ProvisionStreamError::BasinNotFound(e) => Err(e)?, - ProvisionStreamError::StreamAlreadyExists(_) => {} - ProvisionStreamError::Validation(e) => { - unreachable!("auto-create config is validated at the API boundary: {e}") + if should_auto_create(&config) { + if let Err(e) = self + .provision_stream( + basin.clone(), + stream.clone(), + OptionalStreamConfig::default(), + ProvisionMode::CreateOnly { + request_token: None, + }, + ) + .await + { + match e { + ProvisionStreamError::Storage(e) => Err(e)?, + ProvisionStreamError::TransactionConflict(e) => Err(e)?, + ProvisionStreamError::BasinDeletionPending(e) => Err(e)?, + ProvisionStreamError::StreamDeletionPending(e) => Err(e)?, + ProvisionStreamError::BasinNotFound(e) => Err(e)?, + ProvisionStreamError::StreamAlreadyExists(_) => {} + ProvisionStreamError::Validation(_) => { + unreachable!("auto-create uses default config") + } } } + let client = self.streamer_client_guarded(basin, stream).await?; + let encryption = resolve_encryption(client.cipher())?; + Ok(StreamHandle { + db: self.db.clone(), + encryption, + client, + }) + } else { + Err(e.into()) } - self.streamer_client_guarded(basin, stream).await? } - Err(e) => return Err(e.into()), - }; - Ok(StreamHandle { - db: self.db.clone(), - encryption: resolve_encryption(client.cipher())?, - client, - }) - } -} - -/// Which basin setting governs creating a missing stream on demand. -#[derive(Debug, Clone, Copy)] -pub(super) enum AutoCreateOn { - /// `create_stream_on_append` - Append, - /// `create_stream_on_read` - Read, -} - -impl AutoCreateOn { - fn is_enabled(self, config: &BasinConfig) -> bool { - match self { - Self::Append => config.create_stream_on_append, - Self::Read => config.create_stream_on_read, + Err(e) => Err(e.into()), } } } @@ -446,17 +405,15 @@ mod tests { use bytes::Bytes; use s2_common::{ - config::{BasinConfig, OptionalStreamConfig, StreamConfig, TimestampingMode}, + config::{BasinConfig, OptionalStreamConfig, StreamConfig}, record::{Metered, MeteredExt as _, Record, StreamPosition}, resources::ProvisionMode, - stream::{AppendInput, AppendRecord, AppendRecordParts}, }; use s2_storage::record::StoredRecord; use slatedb::{WriteBatch, object_store}; use time::OffsetDateTime; use super::*; - use crate::backend::{error::AppendError, test_util::DbWriteTestExt as _}; async fn new_test_backend() -> Backend { let object_store: Arc = @@ -468,22 +425,6 @@ mod tests { Backend::new(db, ByteSize::b(1)) } - fn append_input(body: &str) -> AppendInput { - let record = - Record::try_from_parts(vec![], Bytes::copy_from_slice(body.as_bytes())).unwrap(); - let record: AppendRecord = AppendRecordParts { - timestamp: None, - record: Metered::from(record), - } - .try_into() - .unwrap(); - AppendInput { - records: vec![record].try_into().unwrap(), - match_seq_num: None, - fencing_token: None, - } - } - #[tokio::test] #[should_panic(expected = "invariant violation: stream `testbasin1/stream1` tail_pos")] async fn start_streamer_fails_if_records_exist_after_tail_pos() { @@ -526,7 +467,7 @@ mod tests { kv::stream_record_data::ser_key(stream_id, record_pos), kv::stream_record_data::ser_value(metered_record.as_ref()), ); - backend.db.write(wb).assert_durable().await; + backend.db.write(wb).await.unwrap(); backend .start_streamer(StreamerGenerationId::next(), basin.clone(), stream.clone()) @@ -535,77 +476,45 @@ mod tests { } #[tokio::test] - async fn streamer_initialization_is_shared_and_survives_cancellation() { - let db = slatedb::Db::builder("test", Arc::new(object_store::memory::InMemory::new())) - .with_settings(slatedb::config::Settings { - flush_interval: None, - ..Default::default() - }) - .build() - .await - .unwrap(); - let backend = Backend::new(db, ByteSize::b(1)); - backend.db.put(b"unflushed", b"value").await.unwrap(); + async fn streamer_client_slot_uses_single_initializer() { + let backend = new_test_backend().await; let basin = BasinName::from_str("testbasin2").unwrap(); let stream = StreamName::from_str("stream2").unwrap(); let slot_1 = backend.streamer_client_slot(&basin, &stream); let slot_2 = backend.streamer_client_slot(&basin, &stream); - let (generation_id_1, generation_id_2, mut future_1, future_2) = match (slot_1, slot_2) { + let (generation_id_1, generation_id_2) = match (slot_1, slot_2) { ( StreamerClientSlot::Initializing { generation_id: generation_id_1, - future: future_1, .. }, StreamerClientSlot::Initializing { generation_id: generation_id_2, - future: future_2, .. }, - ) => (generation_id_1, generation_id_2, future_1, future_2), + ) => (generation_id_1, generation_id_2), _ => panic!("expected both slots to be Initializing"), }; assert_eq!(generation_id_1, generation_id_2); assert_eq!(backend.streamer_slots.len(), 1); - - // Cancel every caller while initialization waits for its snapshot to be durable. - assert!(futures::poll!(&mut future_1).is_pending()); - tokio::task::yield_now().await; - drop((future_1, future_2)); - backend.db.flush().await.unwrap(); - - // The missing stream must still finish initialization and release its snapshot. - tokio::time::timeout(std::time::Duration::from_secs(1), async { - while !backend.streamer_slots.is_empty() { - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - backend.close().await.unwrap(); } #[tokio::test] - async fn config_notification_does_not_start_streamer() { + async fn streamer_client_if_active_is_peek_only() { let backend = new_test_backend().await; let basin = BasinName::from_str("testbasin3").unwrap(); let stream = StreamName::from_str("stream3").unwrap(); - backend.advise_stream_config(&basin, &stream, 1, StreamConfig::default()); - assert!(backend.streamer_slots.is_empty()); - backend.close().await.unwrap(); - } - - #[tokio::test] - async fn initializing_streamer_receives_latest_config() { - let backend = new_test_backend().await; - let basin = BasinName::from_str("config-init").unwrap(); - let stream = StreamName::from_str("stream").unwrap(); - let stream_id = StreamId::new(&basin, &stream); backend - .provision_basin(basin.clone(), BasinConfig::default(), ProvisionMode::Ensure) + .provision_basin( + basin.clone(), + BasinConfig::default(), + ProvisionMode::CreateOnly { + request_token: None, + }, + ) .await .unwrap(); backend @@ -613,40 +522,16 @@ mod tests { basin.clone(), stream.clone(), OptionalStreamConfig::default(), - ProvisionMode::Ensure, + ProvisionMode::CreateOnly { + request_token: None, + }, ) .await .unwrap(); - // Pause initialization after reading metadata, before publishing the client. - let generation_id = StreamerGenerationId::next(); - let client = backend - .start_streamer(generation_id, basin.clone(), stream.clone()) - .await - .unwrap(); - backend.streamer_slots.insert( - stream_id, - StreamerClientSlot::Initializing { - generation_id, - future: futures::future::pending().boxed().shared(), - pending_config: None, - }, - ); - let mut config = StreamConfig::default(); - config.timestamping.mode = TimestampingMode::ClientRequire; - backend.advise_stream_config(&basin, &stream, 20, config); - backend.advise_stream_config(&basin, &stream, 10, StreamConfig::default()); - backend.streamer_finish_initialization(stream_id, generation_id, &Ok(client)); - - let handle = backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) - .await - .unwrap(); - assert!(matches!( - handle.append(append_input("missing timestamp")).await, - Err(AppendError::TimestampMissing(_)) - )); - backend.close().await.unwrap(); + assert!(backend.streamer_slots.is_empty()); + assert!(backend.streamer_client_if_active(&basin, &stream).is_none()); + assert!(backend.streamer_slots.is_empty()); } #[tokio::test] @@ -683,7 +568,6 @@ mod tests { StreamerClientSlot::Initializing { generation_id: current_generation_id, future: future.clone(), - pending_config: None, }, ); @@ -703,6 +587,26 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn concurrent_appends_auto_create_stream_without_spurious_not_found() { + use s2_common::stream::{AppendInput, AppendRecord, AppendRecordParts}; + + use crate::backend::error::AppendError; + + fn append_input(body: &str) -> AppendInput { + let record = + Record::try_from_parts(vec![], Bytes::copy_from_slice(body.as_bytes())).unwrap(); + let record: AppendRecord = AppendRecordParts { + timestamp: None, + record: Metered::from(record), + } + .try_into() + .unwrap(); + AppendInput { + records: vec![record].try_into().unwrap(), + match_seq_num: None, + fencing_token: None, + } + } + let backend = new_test_backend().await; let basin = BasinName::from_str("autocreate").unwrap(); backend @@ -727,9 +631,7 @@ mod tests { let basin = basin.clone(); let stream = stream.clone(); tokio::spawn(async move { - let handle = backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) - .await?; + let handle = backend.open_for_append(&basin, &stream, None).await?; handle.append(append_input(&format!("r{i}"))).await }) }) diff --git a/lite/src/backend/doe.rs b/lite/src/backend/doe.rs deleted file mode 100644 index 37d7ba7a..00000000 --- a/lite/src/backend/doe.rs +++ /dev/null @@ -1,124 +0,0 @@ -//! Transactional scheduling shared by configuration changes, trims, and migration. -//! Appends only postpone eligibility and do not need to update the schedule. - -use std::time::Duration; - -use slatedb::DbTransaction; - -use super::{error::StorageError, kv, store::db_txn_get, timestamp::TimestampSecs}; -use crate::stream_id::StreamId; - -/// Bound polling on active streams, while allowing known expirations to defer -/// checks much farther into the future. -pub(super) const RETRY_INTERVAL: Duration = Duration::from_secs(600); - -pub(super) async fn state( - txn: &DbTransaction, - stream_id: StreamId, -) -> Result, StorageError> { - db_txn_get( - txn, - kv::stream_doe_state::ser_key(stream_id), - kv::stream_doe_state::deser_value, - ) - .await -} - -/// Request a check without adding a second ticket. Even when keeping an earlier -/// ticket, rewrite the state to advance its commit sequence: an in-flight worker -/// must not park or postpone the stream after a concurrent trim/configuration wake. -pub(super) async fn schedule( - txn: &DbTransaction, - stream_id: StreamId, - at: TimestampSecs, -) -> Result<(), StorageError> { - let previous = state(txn, stream_id).await?; - schedule_observed(txn, stream_id, previous, at)?; - Ok(()) -} - -/// Most trims need only the state row. A full trim also restores scheduling for -/// older enabled streams whose last legacy deadline was already consumed. -pub(super) async fn wake_after_trim( - txn: &DbTransaction, - stream_id: StreamId, - has_remaining_records: bool, -) -> Result<(), StorageError> { - let previous = state(txn, stream_id).await?; - if previous.is_none() { - if has_remaining_records { - return Ok(()); - } - let Some((basin, stream)) = db_txn_get( - txn, - kv::stream_id_mapping::ser_key(stream_id), - kv::stream_id_mapping::deser_value, - ) - .await? - else { - return Ok(()); - }; - let Some(meta) = db_txn_get( - txn, - kv::stream_meta::ser_key(&basin, &stream), - kv::stream_meta::deser_value, - ) - .await? - else { - return Ok(()); - }; - if meta.deleted_at.is_some() || meta.config.delete_on_empty.min_age().is_none() { - return Ok(()); - } - } - schedule_observed(txn, stream_id, previous, TimestampSecs::now())?; - Ok(()) -} - -fn schedule_observed( - txn: &DbTransaction, - stream_id: StreamId, - previous: Option, - at: TimestampSecs, -) -> Result<(), slatedb::Error> { - let next = match previous { - Some(kv::stream_doe_state::State::Scheduled(check)) if check.at <= at => { - kv::stream_doe_state::State::Scheduled(check) - } - _ => kv::stream_doe_state::State::Scheduled(kv::stream_doe_state::Check { - at, - id: rand::random(), - }), - }; - replace(txn, stream_id, previous, Some(next)) -} - -pub(super) async fn clear(txn: &DbTransaction, stream_id: StreamId) -> Result<(), StorageError> { - let previous = state(txn, stream_id).await?; - replace(txn, stream_id, previous, None)?; - Ok(()) -} - -/// The caller must have read the state in this serializable transaction. -pub(super) fn replace( - txn: &DbTransaction, - stream_id: StreamId, - previous: Option, - next: Option, -) -> Result<(), slatedb::Error> { - if previous != next { - if let Some(kv::stream_doe_state::State::Scheduled(check)) = previous { - txn.delete(kv::stream_doe_check::ser_key(stream_id, check))?; - } - if let Some(kv::stream_doe_state::State::Scheduled(check)) = next { - txn.put(kv::stream_doe_check::ser_key(stream_id, check), [])?; - } - } - let key = kv::stream_doe_state::ser_key(stream_id); - match next { - Some(next) => txn.put(key, kv::stream_doe_state::ser_value(next))?, - None if previous.is_some() => txn.delete(key)?, - None => (), - } - Ok(()) -} diff --git a/lite/src/backend/error.rs b/lite/src/backend/error.rs index 62970c58..3c441a65 100644 --- a/lite/src/backend/error.rs +++ b/lite/src/backend/error.rs @@ -12,20 +12,12 @@ use crate::backend::kv; #[derive(Debug, Clone, thiserror::Error)] pub enum StorageError { - #[error("invariant violation: {0}")] - InvariantViolation(String), #[error("deserialization: {0}")] Deserialization(#[from] kv::DeserializationError), #[error("database: {0}")] Database(Arc), } -impl StorageError { - pub(super) fn is_transaction_conflict(&self) -> bool { - matches!(self, Self::Database(err) if err.kind() == slatedb::ErrorKind::Transaction) - } -} - impl From for StorageError { fn from(error: slatedb::Error) -> Self { StorageError::Database(Arc::new(error)) @@ -114,10 +106,8 @@ pub(super) enum AppendErrorInternal { StreamerMissingInActionError(#[from] StreamerMissingInActionError), #[error(transparent)] RequestDroppedError(#[from] RequestDroppedError), - #[error("stream deletion pending")] - StreamDeletionPending { - durability_dependency: RangeTo, - }, + #[error(transparent)] + StreamDeletionPending(#[from] StreamDeletionPendingError), #[error(transparent)] ConditionFailed(#[from] AppendConditionFailedError), #[error(transparent)] @@ -129,9 +119,6 @@ pub(super) enum AppendErrorInternal { impl AppendErrorInternal { pub fn durability_dependency(&self) -> RangeTo { match self { - Self::StreamDeletionPending { - durability_dependency, - } => *durability_dependency, Self::ConditionFailed(e) => e.durability_dependency(), Self::MaxSeqNum(e) => e.durability_dependency(), _ => ..0, @@ -203,9 +190,7 @@ impl From for AppendError { AppendError::StreamerMissingInActionError(e) } AppendErrorInternal::RequestDroppedError(e) => AppendError::RequestDroppedError(e), - AppendErrorInternal::StreamDeletionPending { .. } => { - AppendError::StreamDeletionPending(StreamDeletionPendingError) - } + AppendErrorInternal::StreamDeletionPending(e) => AppendError::StreamDeletionPending(e), AppendErrorInternal::ConditionFailed(e) => AppendError::ConditionFailed(e), AppendErrorInternal::TimestampMissing(e) => AppendError::TimestampMissing(e), AppendErrorInternal::MaxSeqNum(e) => AppendError::MaxSeqNum(e), @@ -323,7 +308,7 @@ impl From for ListStreamsError { #[derive(Debug, Clone, thiserror::Error)] pub enum ProvisionStreamError { #[error(transparent)] - Storage(StorageError), + Storage(#[from] StorageError), #[error(transparent)] TransactionConflict(#[from] TransactionConflictError), #[error(transparent)] @@ -340,16 +325,10 @@ pub enum ProvisionStreamError { impl From for ProvisionStreamError { fn from(err: slatedb::Error) -> Self { - Self::from(StorageError::from(err)) - } -} - -impl From for ProvisionStreamError { - fn from(err: StorageError) -> Self { - if err.is_transaction_conflict() { + if err.kind() == slatedb::ErrorKind::Transaction { Self::TransactionConflict(TransactionConflictError) } else { - Self::Storage(err) + Self::Storage(err.into()) } } } @@ -413,18 +392,6 @@ pub enum StreamDeleteOnEmptyError { DeleteStream(#[from] DeleteStreamError), } -impl StreamDeleteOnEmptyError { - pub(super) fn is_transaction_conflict(&self) -> bool { - match self { - Self::Storage(err) | Self::DeleteStream(DeleteStreamError::Storage(err)) => { - err.is_transaction_conflict() - } - Self::DeleteStream(DeleteStreamError::TransactionConflict(_)) => true, - _ => false, - } - } -} - #[derive(Debug, Clone, thiserror::Error)] pub enum ListBasinsError { #[error(transparent)] @@ -498,7 +465,7 @@ impl From for ReconfigureBasinError { #[derive(Debug, Clone, thiserror::Error)] pub enum ReconfigureStreamError { #[error(transparent)] - Storage(StorageError), + Storage(#[from] StorageError), #[error(transparent)] TransactionConflict(#[from] TransactionConflictError), #[error(transparent)] @@ -515,16 +482,10 @@ pub enum ReconfigureStreamError { impl From for ReconfigureStreamError { fn from(err: slatedb::Error) -> Self { - Self::from(StorageError::from(err)) - } -} - -impl From for ReconfigureStreamError { - fn from(err: StorageError) -> Self { - if err.is_transaction_conflict() { + if err.kind() == slatedb::ErrorKind::Transaction { Self::TransactionConflict(TransactionConflictError) } else { - Self::Storage(err) + Self::Storage(err.into()) } } } diff --git a/lite/src/backend/kv/mod.rs b/lite/src/backend/kv/mod.rs index 95f0e505..893abe27 100644 --- a/lite/src/backend/kv/mod.rs +++ b/lite/src/backend/kv/mod.rs @@ -1,8 +1,6 @@ pub mod basin_deletion_pending; pub mod basin_meta; -pub mod stream_doe_check; pub mod stream_doe_deadline; -pub mod stream_doe_state; pub mod stream_fencing_token; pub mod stream_id_mapping; pub mod stream_meta; @@ -10,6 +8,7 @@ pub mod stream_record_data; pub mod stream_record_timestamp; pub mod stream_tail_position; pub mod stream_trim_point; +pub mod timestamp; use std::{ops::Range, str::FromStr}; @@ -20,7 +19,7 @@ use s2_common::{ use strum::FromRepr; use thiserror::Error; -use crate::{backend::timestamp::TimestampSecs, stream_id::StreamId}; +use crate::stream_id::StreamId; #[derive(Debug, Clone, Error)] pub enum DeserializationError { @@ -50,8 +49,6 @@ pub enum KeyType { StreamRecordData = 6, StreamRecordTimestamp = 7, StreamDeleteOnEmptyDeadline = 10, - StreamDeleteOnEmptyState = 11, - StreamDeleteOnEmptyCheck = 12, } #[derive(Debug, Clone)] @@ -92,21 +89,10 @@ pub enum Key { /// Key: StreamID Timestamp SeqNum /// Value: empty StreamRecordTimestamp(StreamId, StreamPosition), - /// (SDOED) legacy schedule, consumed only to initialize the new DOE state - /// Key: TimestampSecs StreamID ScheduleID (u128, absent in legacy keys) + /// (SDOED) per-deadline-per-stream, deletable, present while pending + /// Key: TimestampSecs StreamID /// Value: MinAge seconds (u64) - StreamDeleteOnEmptyDeadline(TimestampSecs, StreamId, Option), - /// (SDOES) per-stream, updatable, optional - /// Key: StreamID - /// Value: Tag (u8: 0 = parked, 1 = scheduled). - /// Scheduled values append TimestampSecs (u32) CheckID (u128). - /// Uses the entry's SlateDB sequence as its revision. - /// State older than the current stream-ID mapping is stale. - StreamDeleteOnEmptyState(StreamId), - /// (SDOEC) per-check, immutable, deletable, time-ordered index of scheduled states - /// Key: TimestampSecs (u32) StreamID CheckID (u128) - /// Value: empty - StreamDeleteOnEmptyCheck(StreamId, stream_doe_state::Check), + StreamDeleteOnEmptyDeadline(timestamp::TimestampSecs, StreamId), } impl From for Bytes { @@ -123,12 +109,8 @@ impl From for Bytes { Key::StreamRecordTimestamp(stream_id, pos) => { stream_record_timestamp::ser_key(stream_id, pos) } - Key::StreamDeleteOnEmptyDeadline(deadline, stream_id, schedule_id) => { - stream_doe_deadline::ser_key(deadline, stream_id, schedule_id) - } - Key::StreamDeleteOnEmptyState(stream_id) => stream_doe_state::ser_key(stream_id), - Key::StreamDeleteOnEmptyCheck(stream_id, check) => { - stream_doe_check::ser_key(stream_id, check) + Key::StreamDeleteOnEmptyDeadline(deadline, stream_id) => { + stream_doe_deadline::ser_key(deadline, stream_id) } } } @@ -165,16 +147,8 @@ impl TryFrom for Key { .map(|(stream_id, pos)| Key::StreamRecordData(stream_id, pos)), KeyType::StreamRecordTimestamp => stream_record_timestamp::deser_key(bytes) .map(|(stream_id, pos)| Key::StreamRecordTimestamp(stream_id, pos)), - KeyType::StreamDeleteOnEmptyDeadline => { - stream_doe_deadline::deser_key(bytes).map(|(deadline, stream_id, schedule_id)| { - Key::StreamDeleteOnEmptyDeadline(deadline, stream_id, schedule_id) - }) - } - KeyType::StreamDeleteOnEmptyState => { - stream_doe_state::deser_key(bytes).map(Key::StreamDeleteOnEmptyState) - } - KeyType::StreamDeleteOnEmptyCheck => stream_doe_check::deser_key(bytes) - .map(|(stream_id, check)| Key::StreamDeleteOnEmptyCheck(stream_id, check)), + KeyType::StreamDeleteOnEmptyDeadline => stream_doe_deadline::deser_key(bytes) + .map(|(deadline, stream_id)| Key::StreamDeleteOnEmptyDeadline(deadline, stream_id)), } } } diff --git a/lite/src/backend/kv/stream_doe_check.rs b/lite/src/backend/kv/stream_doe_check.rs deleted file mode 100644 index 2ec02c2c..00000000 --- a/lite/src/backend/kv/stream_doe_check.rs +++ /dev/null @@ -1,78 +0,0 @@ -use std::ops::Range; - -use bytes::{Buf, BufMut, Bytes, BytesMut}; - -use super::{DeserializationError, KeyType, check_exact_size, stream_doe_state::Check}; -use crate::{backend::timestamp::TimestampSecs, stream_id::StreamId}; - -const KEY_LEN: usize = 1 + 4 + StreamId::LEN + 16; - -pub fn ser_key(stream_id: StreamId, check: Check) -> Bytes { - let mut buf = BytesMut::with_capacity(KEY_LEN); - buf.put_u8(KeyType::StreamDeleteOnEmptyCheck as u8); - buf.put_u32(check.at.as_u32()); - buf.put_slice(stream_id.as_bytes()); - buf.put_u128(check.id); - buf.freeze() -} - -pub fn deser_key(mut bytes: Bytes) -> Result<(StreamId, Check), DeserializationError> { - check_exact_size(&bytes, KEY_LEN)?; - let ordinal = bytes.get_u8(); - if ordinal != KeyType::StreamDeleteOnEmptyCheck as u8 { - return Err(DeserializationError::InvalidOrdinal(ordinal)); - } - let at = TimestampSecs::from_secs(bytes.get_u32()); - let mut stream_id = [0; StreamId::LEN]; - bytes.copy_to_slice(&mut stream_id); - let id = bytes.get_u128(); - Ok((stream_id.into(), Check { at, id })) -} - -pub fn due_key_range(now: TimestampSecs) -> Range { - let start = Bytes::from_static(&[KeyType::StreamDeleteOnEmptyCheck as u8]); - let mut end = BytesMut::with_capacity(5); - end.put_u8(KeyType::StreamDeleteOnEmptyCheck as u8); - end.put_u32(now.as_u32()); - start..super::increment_bytes(end).expect("non-empty") -} - -#[cfg(test)] -mod tests { - use proptest::prelude::*; - - use super::*; - - proptest! { - #[test] - fn roundtrip_check(at in any::(), id in any::(), stream in any::<[u8; StreamId::LEN]>()) { - let check = Check { at: TimestampSecs::from_secs(at), id }; - let stream_id = StreamId::from(stream); - let key = ser_key(stream_id, check); - prop_assert_eq!(deser_key(key.clone()).unwrap(), (stream_id, check)); - prop_assert_eq!(Bytes::from(super::super::Key::try_from(key.clone()).unwrap()), key); - } - } - - #[test] - fn due_range_includes_every_ticket_at_boundary() { - for seconds in [0, 100, u32::MAX] { - let at = TimestampSecs::from_secs(seconds); - let range = due_key_range(at); - for stream_id in [[0; StreamId::LEN], [u8::MAX; StreamId::LEN]].map(StreamId::from) { - for id in [0, u128::MAX] { - assert!(range.contains(&ser_key(stream_id, Check { at, id }))); - if let Some(next) = seconds.checked_add(1) { - assert!(!range.contains(&ser_key( - stream_id, - Check { - at: TimestampSecs::from_secs(next), - id - } - ))); - } - } - } - } - } -} diff --git a/lite/src/backend/kv/stream_doe_deadline.rs b/lite/src/backend/kv/stream_doe_deadline.rs index 8eef06a3..573c5217 100644 --- a/lite/src/backend/kv/stream_doe_deadline.rs +++ b/lite/src/backend/kv/stream_doe_deadline.rs @@ -1,35 +1,48 @@ +use std::{ops::Range, time::Duration}; + use bytes::{Buf, BufMut, Bytes, BytesMut}; -use super::{DeserializationError, KeyType, check_exact_size}; -use crate::{backend::timestamp::TimestampSecs, stream_id::StreamId}; +use super::{DeserializationError, KeyType, check_exact_size, timestamp::TimestampSecs}; +use crate::stream_id::StreamId; -const LEGACY_KEY_LEN: usize = 1 + 4 + StreamId::LEN; -const KEY_LEN: usize = LEGACY_KEY_LEN + 16; -/// Legacy keys are retained only for decoding and migration. New schedules use -/// `stream_doe_state` and `stream_doe_check`. -pub fn ser_key(deadline: TimestampSecs, stream_id: StreamId, schedule_id: Option) -> Bytes { - let key_len = if schedule_id.is_some() { - KEY_LEN - } else { - LEGACY_KEY_LEN - }; - let mut buf = BytesMut::with_capacity(key_len); +const KEY_LEN: usize = 1 + 4 + StreamId::LEN; +const VALUE_LEN: usize = 8; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(in crate::backend) struct Entry { + pub deadline: TimestampSecs, + pub min_age: Duration, +} + +impl Entry { + pub fn last_write_cutoff(self) -> Option { + self.deadline.checked_sub_duration(self.min_age) + } +} + +pub fn ser_key(deadline: TimestampSecs, stream_id: StreamId) -> Bytes { + let mut buf = BytesMut::with_capacity(KEY_LEN); buf.put_u8(KeyType::StreamDeleteOnEmptyDeadline as u8); buf.put_u32(deadline.as_u32()); buf.put_slice(stream_id.as_bytes()); - if let Some(schedule_id) = schedule_id { - buf.put_u128(schedule_id); - } - debug_assert_eq!(buf.len(), key_len, "serialized length mismatch"); + debug_assert_eq!(buf.len(), KEY_LEN, "serialized length mismatch"); buf.freeze() } -pub fn deser_key( - mut bytes: Bytes, -) -> Result<(TimestampSecs, StreamId, Option), DeserializationError> { - if bytes.len() != LEGACY_KEY_LEN { - check_exact_size(&bytes, KEY_LEN)?; - } +pub fn expired_key_range(deadline: TimestampSecs) -> Range { + let start = Bytes::from(vec![KeyType::StreamDeleteOnEmptyDeadline as u8]); + let end = ser_key_range_end(deadline); + start..end +} + +fn ser_key_range_end(deadline: TimestampSecs) -> Bytes { + let max_stream_id = StreamId::from([u8::MAX; StreamId::LEN]); + let end_key = ser_key(deadline, max_stream_id); + super::increment_bytes(BytesMut::from(end_key.as_ref())).expect("non-empty") +} + +pub fn deser_key(mut bytes: Bytes) -> Result<(TimestampSecs, StreamId), DeserializationError> { + check_exact_size(&bytes, KEY_LEN)?; let ordinal = bytes.get_u8(); if ordinal != (KeyType::StreamDeleteOnEmptyDeadline as u8) { return Err(DeserializationError::InvalidOrdinal(ordinal)); @@ -37,20 +50,32 @@ pub fn deser_key( let deadline_secs = bytes.get_u32(); let mut stream_id_bytes = [0u8; StreamId::LEN]; bytes.copy_to_slice(&mut stream_id_bytes); - let schedule_id = bytes.has_remaining().then(|| bytes.get_u128()); Ok(( TimestampSecs::from_secs(deadline_secs), stream_id_bytes.into(), - schedule_id, )) } +pub fn ser_value(min_age: Duration) -> Bytes { + let mut buf = BytesMut::with_capacity(VALUE_LEN); + buf.put_u64(min_age.as_secs()); + debug_assert_eq!(buf.len(), VALUE_LEN, "serialized length mismatch"); + buf.freeze() +} + +pub fn deser_value(mut bytes: Bytes) -> Result { + check_exact_size(&bytes, VALUE_LEN)?; + Ok(Duration::from_secs(bytes.get_u64())) +} + #[cfg(test)] mod tests { + use std::time::Duration; + use proptest::prelude::*; use crate::{ - backend::{kv::stream_doe_deadline, timestamp::TimestampSecs}, + backend::kv::{stream_doe_deadline, timestamp::TimestampSecs}, stream_id::StreamId, }; @@ -59,17 +84,21 @@ mod tests { fn roundtrip_stream_doe_deadline_key( deadline_secs in any::(), stream_id_bytes in any::<[u8; StreamId::LEN]>(), - schedule_id in proptest::option::of(any::()), ) { let deadline = TimestampSecs::from_secs(deadline_secs); let stream_id = StreamId::from(stream_id_bytes); - let bytes = stream_doe_deadline::ser_key(deadline, stream_id, schedule_id); - let (decoded_deadline, decoded_stream_id, decoded_schedule_id) = stream_doe_deadline::deser_key(bytes.clone()).unwrap(); + let bytes = stream_doe_deadline::ser_key(deadline, stream_id); + let (decoded_deadline, decoded_stream_id) = stream_doe_deadline::deser_key(bytes).unwrap(); prop_assert_eq!(deadline, decoded_deadline); prop_assert_eq!(stream_id, decoded_stream_id); - prop_assert_eq!(schedule_id, decoded_schedule_id); - let decoded = super::super::Key::try_from(bytes.clone()).unwrap(); - prop_assert_eq!(bytes, bytes::Bytes::from(decoded)); } } + + #[test] + fn roundtrip_stream_doe_deadline_value() { + let min_age = Duration::from_secs(123); + let bytes = stream_doe_deadline::ser_value(min_age); + let decoded = stream_doe_deadline::deser_value(bytes).unwrap(); + assert_eq!(min_age, decoded); + } } diff --git a/lite/src/backend/kv/stream_doe_state.rs b/lite/src/backend/kv/stream_doe_state.rs deleted file mode 100644 index e1f07b49..00000000 --- a/lite/src/backend/kv/stream_doe_state.rs +++ /dev/null @@ -1,82 +0,0 @@ -use bytes::{Buf, BufMut, Bytes, BytesMut}; - -use super::{DeserializationError, KeyType, check_exact_size}; -use crate::{backend::timestamp::TimestampSecs, stream_id::StreamId}; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Check { - pub at: TimestampSecs, - pub id: u128, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum State { - Scheduled(Check), - /// An observed record has no expiration. Only a trim can remove it. - Parked, -} - -pub fn ser_key(stream_id: StreamId) -> Bytes { - super::ser_stream_id_key(KeyType::StreamDeleteOnEmptyState, stream_id) -} - -pub fn deser_key(bytes: Bytes) -> Result { - super::deser_stream_id_key(KeyType::StreamDeleteOnEmptyState, bytes) -} - -pub fn ser_value(state: State) -> Bytes { - let mut buf = BytesMut::with_capacity(21); - match state { - State::Parked => buf.put_u8(0), - State::Scheduled(Check { at, id }) => { - buf.put_u8(1); - buf.put_u32(at.as_u32()); - buf.put_u128(id); - } - } - buf.freeze() -} - -pub fn deser_value(mut bytes: Bytes) -> Result { - super::check_min_size(&bytes, 1)?; - match bytes.get_u8() { - 0 => { - check_exact_size(&bytes, 0)?; - Ok(State::Parked) - } - 1 => { - check_exact_size(&bytes, 20)?; - Ok(State::Scheduled(Check { - at: TimestampSecs::from_secs(bytes.get_u32()), - id: bytes.get_u128(), - })) - } - ordinal => Err(DeserializationError::InvalidOrdinal(ordinal)), - } -} - -#[cfg(test)] -mod tests { - use proptest::prelude::*; - - use super::*; - - proptest! { - #[test] - fn roundtrip_state(at in any::(), id in any::(), stream in any::<[u8; StreamId::LEN]>()) { - for state in [State::Parked, State::Scheduled(Check { at: TimestampSecs::from_secs(at), id })] { - prop_assert_eq!(deser_value(ser_value(state)).unwrap(), state); - } - let key = ser_key(stream.into()); - prop_assert_eq!(deser_key(key.clone()).unwrap(), StreamId::from(stream)); - prop_assert_eq!(Bytes::from(super::super::Key::try_from(key.clone()).unwrap()), key); - } - } - - #[test] - fn reject_truncated_or_unknown_state() { - for bytes in [&[][..], &[0, 1], &[1], &[2]] { - assert!(deser_value(Bytes::copy_from_slice(bytes)).is_err()); - } - } -} diff --git a/lite/src/backend/kv/stream_meta.rs b/lite/src/backend/kv/stream_meta.rs index f655aaa7..70e22a80 100644 --- a/lite/src/backend/kv/stream_meta.rs +++ b/lite/src/backend/kv/stream_meta.rs @@ -16,7 +16,6 @@ use super::{ DeserializationError, KeyType, check_min_size, deser_json_value, increment_bytes, invalid_value_err, ser_json_value, }; -use crate::backend::resolve_stream_config; const FIELD_SEPARATOR: u8 = b'\0'; @@ -57,12 +56,12 @@ impl TryFrom for StreamMeta { fn try_from(serde: StreamMetaSerde) -> Result { let config = match serde.config { - Some(api_config) => OptionalStreamConfig::try_from(api_config)?, - None => OptionalStreamConfig::default(), + Some(api_config) => OptionalStreamConfig::try_from(api_config)?.into(), + None => StreamConfig::default(), }; Ok(Self { - config: resolve_stream_config(config, OptionalStreamConfig::default()), + config, cipher: serde.cipher, created_at: serde.created_at, deleted_at: serde.deleted_at, @@ -156,7 +155,7 @@ mod tests { use proptest::prelude::*; use s2_common::{ basin::BasinName, - config::{OptionalDeleteOnEmptyConfig, OptionalStreamConfig, StreamConfig}, + config::{OptionalDeleteOnEmptyConfig, OptionalStreamConfig, StorageClass, StreamConfig}, encryption::EncryptionAlgorithm, stream::{StreamName, StreamNamePrefix, StreamNameStartAfter}, }; @@ -168,7 +167,7 @@ mod tests { #[test] fn value_roundtrip_stream_meta() { let config = OptionalStreamConfig { - storage_class: Some("express".into()), + storage_class: Some(StorageClass::Express), delete_on_empty: OptionalDeleteOnEmptyConfig { min_age: Some(Duration::ZERO), }, @@ -229,7 +228,7 @@ mod tests { let decoded = super::deser_value(bytes).unwrap(); let default_config = StreamConfig::default(); - assert_eq!(decoded.config.storage_class.as_deref(), Some("express")); + assert_eq!(decoded.config.storage_class, default_config.storage_class); assert_eq!( decoded.config.retention_policy, default_config.retention_policy diff --git a/lite/src/backend/timestamp.rs b/lite/src/backend/kv/timestamp.rs similarity index 91% rename from lite/src/backend/timestamp.rs rename to lite/src/backend/kv/timestamp.rs index 1455cac7..ae368b8e 100644 --- a/lite/src/backend/timestamp.rs +++ b/lite/src/backend/kv/timestamp.rs @@ -44,14 +44,6 @@ impl TimestampSecs { .map(|secs| Self(secs as u32)) } - pub fn saturating_add_duration(self, dur: Duration) -> Self { - Self( - u64::from(self.0) - .saturating_add(dur.as_secs()) - .min(u64::from(u32::MAX)) as u32, - ) - } - fn from_system_time(time: SystemTime) -> Self { match time.duration_since(UNIX_EPOCH) { Ok(duration) => { diff --git a/lite/src/backend/mod.rs b/lite/src/backend/mod.rs index 99aab0fb..be8db24f 100644 --- a/lite/src/backend/mod.rs +++ b/lite/src/backend/mod.rs @@ -1,23 +1,15 @@ -use s2_common::{ - config::{OptionalStreamConfig, StreamConfig}, - encryption::EncryptionSpec, -}; +use s2_common::encryption::EncryptionSpec; pub mod error; mod basins; pub mod bgtasks; mod core; -mod doe; mod durability_notifier; mod read; mod store; mod streamer; mod streams; -mod timestamp; - -#[cfg(test)] -mod test_util; mod append; mod kv; @@ -33,12 +25,3 @@ pub struct StreamHandle { } pub const FOLLOWER_MAX_LAG: usize = 25; - -fn resolve_stream_config( - config: OptionalStreamConfig, - basin_defaults: OptionalStreamConfig, -) -> StreamConfig { - let mut config = config.merge(basin_defaults); - config.storage_class.get_or_insert_with(|| "express".into()); - config -} diff --git a/lite/src/backend/read.rs b/lite/src/backend/read.rs index 3ff6c5bc..7043409f 100644 --- a/lite/src/backend/read.rs +++ b/lite/src/backend/read.rs @@ -4,7 +4,6 @@ use futures::{Stream, StreamExt as _}; use s2_common::{ basin::BasinName, caps, - config::OptionalStreamConfig, encryption::{EncryptionKey, EncryptionSpec}, read_extent::{EvaluatedReadLimit, ReadLimit, ReadUntil}, record::{Metered, MeteredSize as _, SeqNum, StreamPosition, Timestamp}, @@ -16,7 +15,7 @@ use s2_storage::record::{ use slatedb::config::{DurabilityLevel, ScanOptions}; use tokio::{sync::broadcast, time::Instant}; -use super::{Backend, StreamHandle, core::AutoCreateOn}; +use super::{Backend, StreamHandle}; use crate::{ backend::{ error::{ @@ -29,7 +28,6 @@ use crate::{ }; impl Backend { - /// Open a stream for a check tail. pub async fn open_for_check_tail( &self, basin: &BasinName, @@ -38,29 +36,22 @@ impl Backend { self.stream_handle_with_auto_create::( basin, stream, - AutoCreateOn::Read, - OptionalStreamConfig::default(), + |config| config.create_stream_on_read, |_| Ok(EncryptionSpec::Plain), ) .await } - /// Open a stream for a read or read session. - /// - /// `stream_config` is applied if the stream is created on read. Unset fields inherit the - /// basin's default stream configuration. Ignored if the stream already exists. pub async fn open_for_read( &self, basin: &BasinName, stream: &StreamName, encryption_key: Option, - stream_config: OptionalStreamConfig, ) -> Result { self.stream_handle_with_auto_create::( basin, stream, - AutoCreateOn::Read, - stream_config, + |config| config.create_stream_on_read, |cipher| Ok(EncryptionSpec::resolve(cipher, encryption_key)?), ) .await @@ -106,10 +97,6 @@ async fn read_session( start: ReadStart, end: ReadEnd, ) -> Result> + 'static, ReadError> { - // An exhausted limit completes even when the requested start is unwritten. - if end.limit.remaining(0, 0) == EvaluatedReadLimit::Exhausted { - return Ok(futures::stream::empty().left_stream()); - } let stream_id = client.stream_id(); let tail = client.check_tail().await?; let mut state = ReadSessionState { @@ -253,7 +240,7 @@ async fn read_session( } } }; - Ok(session.right_stream()) + Ok(session) } async fn read_start_seq_num( @@ -280,9 +267,13 @@ async fn read_start_seq_num( return Err(UnwrittenError(tail).into()); } } - // Resolve to a sequence number before deciding whether the read starts at the tail, so a - // timestamp start that resolves to the tail is treated like a sequence number start there. - let start_seq_num = match read_pos { + if let ReadPosition::SeqNum(start_seq_num) = read_pos + && start_seq_num == tail.seq_num + && !end.may_follow() + { + return Err(UnwrittenError(tail).into()); + } + Ok(match read_pos { ReadPosition::SeqNum(start_seq_num) => start_seq_num, ReadPosition::Timestamp(start_timestamp) => { resolve_timestamp(db, stream_id, start_timestamp) @@ -290,11 +281,7 @@ async fn read_start_seq_num( .unwrap_or(tail) .seq_num } - }; - if start_seq_num == tail.seq_num && !end.may_follow() { - return Err(UnwrittenError(tail).into()); - } - Ok(start_seq_num) + }) } async fn resolve_timestamp( @@ -434,9 +421,7 @@ mod tests { use super::*; use crate::{ - backend::{ - FOLLOWER_MAX_LAG, kv, streamer::DORMANT_TIMEOUT, test_util::DbWriteTestExt as _, - }, + backend::{FOLLOWER_MAX_LAG, kv, streamer::DORMANT_TIMEOUT}, stream_id::StreamId, }; @@ -511,8 +496,8 @@ mod tests { ), kv::stream_record_timestamp::ser_value(), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( @@ -525,8 +510,8 @@ mod tests { ), kv::stream_record_timestamp::ser_value(), ) - .assert_durable() - .await; + .await + .unwrap(); // Should find record in stream_a let result = resolve_timestamp(&backend.db, stream_a, 500).await.unwrap(); @@ -577,7 +562,7 @@ mod tests { let input = append_input(Record::try_from_parts(vec![], bytes::Bytes::from("x")).unwrap()); let ack = backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_append(&basin, &stream, None) .await .unwrap() .append(input) @@ -588,7 +573,7 @@ mod tests { let stream_id = StreamId::new(&basin, &stream); let mut batch = WriteBatch::new(); batch.delete(kv::stream_record_data::ser_key(stream_id, ack.start)); - backend.db.write(batch).assert_durable().await; + backend.db.write(batch).await.unwrap(); let start = ReadStart { from: ReadFrom::SeqNum(0), @@ -600,7 +585,7 @@ mod tests { wait: None, }; let session = backend - .open_for_read(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_read(&basin, &stream, None) .await .unwrap() .read(start, end) @@ -657,7 +642,7 @@ mod tests { }; let session = backend - .open_for_read(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_read(&basin, &stream, None) .await .unwrap() .read(start, end) @@ -731,7 +716,7 @@ mod tests { let initial_input = append_input(Record::try_from_parts(vec![], bytes::Bytes::from("initial")).unwrap()); backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_append(&basin, &stream, None) .await .unwrap() .append(initial_input) @@ -751,7 +736,7 @@ mod tests { }; let session = backend - .open_for_read(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_read(&basin, &stream, None) .await .unwrap() .read(start, end) @@ -791,7 +776,7 @@ mod tests { let follow_input = append_input(Record::try_from_parts(vec![], bytes::Bytes::from("follow-1")).unwrap()); backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_append(&basin, &stream, None) .await .unwrap() .append(follow_input) @@ -896,7 +881,7 @@ mod tests { wait: Some(wait), }; let session = backend - .open_for_read(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_read(&basin, &stream, None) .await .unwrap() .read(start, end) @@ -921,7 +906,7 @@ mod tests { Record::try_from_parts(vec![], bytes::Bytes::from(format!("lagged-{i}"))).unwrap(), ); let ack = backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_append(&basin, &stream, None) .await .unwrap() .append(input) @@ -930,7 +915,7 @@ mod tests { delete_batch.delete(kv::stream_record_data::ser_key(stream_id, ack.start)); } - backend.db.write(delete_batch).assert_durable().await; + backend.db.write(delete_batch).await.unwrap(); tokio::time::advance(wait + Duration::from_secs(1)).await; tokio::task::yield_now().await; @@ -975,7 +960,7 @@ mod tests { let initial_input = append_input(Record::try_from_parts(vec![], bytes::Bytes::from("initial")).unwrap()); backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_append(&basin, &stream, None) .await .unwrap() .append(initial_input) @@ -992,7 +977,7 @@ mod tests { wait: None, }; let session = backend - .open_for_read(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_read(&basin, &stream, None) .await .unwrap() .read(start, end) @@ -1022,7 +1007,7 @@ mod tests { let follow_input = append_input(Record::try_from_parts(vec![], bytes::Bytes::from("follow-1")).unwrap()); backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_append(&basin, &stream, None) .await .unwrap() .append(follow_input) diff --git a/lite/src/backend/store.rs b/lite/src/backend/store.rs index 6b26bcdb..791bc561 100644 --- a/lite/src/backend/store.rs +++ b/lite/src/backend/store.rs @@ -1,6 +1,6 @@ use bytes::Bytes; use slatedb::{ - DbSnapshot, DbTransaction, KeyValue, + DbTransaction, config::{DurabilityLevel, ReadOptions}, }; @@ -19,15 +19,6 @@ impl Backend { &self, key: K, deser: impl FnOnce(Bytes) -> Result, - ) -> Result, StorageError> { - self.db_get_with(key, |entry| deser(entry.value)).await - } - - /// Read a remotely durable row, including its sequence and timestamps. - pub(super) async fn db_get_with + Send, V>( - &self, - key: K, - deser: impl FnOnce(KeyValue) -> Result, ) -> Result, StorageError> { let read_opts = ReadOptions { durability_filter: DurabilityLevel::Remote, @@ -35,7 +26,7 @@ impl Backend { }; let value = self .db - .get_key_value_with_options(key, &read_opts) + .get_with_options(key, &read_opts) .await? .map(deser) .transpose()?; @@ -43,40 +34,11 @@ impl Backend { } } -pub(super) async fn db_snapshot_get_with + Send, V>( - snapshot: &DbSnapshot, - key: K, - deser: impl FnOnce(KeyValue) -> Result, -) -> Result, StorageError> { - Ok(snapshot.get_key_value(key).await?.map(deser).transpose()?) -} - pub(super) async fn db_txn_get + Send, V>( txn: &DbTransaction, key: K, deser: impl FnOnce(Bytes) -> Result, ) -> Result, StorageError> { - db_txn_get_with(txn, key, |entry| deser(entry.value)).await -} - -/// Read a transaction row, which may not yet be durable. -pub(super) async fn db_txn_get_with + Send, V>( - txn: &DbTransaction, - key: K, - deser: impl FnOnce(KeyValue) -> Result, -) -> Result, StorageError> { - let value = txn.get_key_value(key).await?.map(deser).transpose()?; + let value = txn.get(key).await?.map(deser).transpose()?; Ok(value) } - -/// Commit metadata changes and wait until remote reads can observe them. -/// Return the committed sequence number, or `None` if there were no writes. -pub(super) async fn db_txn_commit_durable( - txn: DbTransaction, -) -> Result, slatedb::Error> { - let Some(handle) = txn.commit().await? else { - return Ok(None); - }; - handle.await_durable().await?; - Ok(Some(handle.seqnum())) -} diff --git a/lite/src/backend/streamer.rs b/lite/src/backend/streamer.rs index 1dd567e8..d8acb6a3 100644 --- a/lite/src/backend/streamer.rs +++ b/lite/src/backend/streamer.rs @@ -28,7 +28,7 @@ use s2_storage::record::{ }; use slatedb::{ IterationOrder, WriteBatch, - config::{PutOptions, ScanOptions, Ttl}, + config::{PutOptions, ScanOptions, Ttl, WriteOptions}, }; use tokio::{ sync::{Semaphore, SemaphorePermit, broadcast, mpsc, oneshot}, @@ -40,21 +40,27 @@ use crate::{ backend::{ append, bgtasks::BgtaskTrigger, - doe, durability_notifier::DurabilityNotifier, error::{ AppendConditionFailedError, AppendErrorInternal, AppendTimestampRequiredError, DeleteStreamError, MaxSeqNumError, RequestDroppedError, StorageError, - StreamerMissingInActionError, + StreamDeletionPendingError, StreamerMissingInActionError, }, kv, - timestamp::TimestampSecs, }, metrics, stream_id::StreamId, }; pub(super) const DORMANT_TIMEOUT: Duration = Duration::from_secs(60); +// Rate-limit delete-on-empty scheduling and pad deadlines to cover the period. +const DOE_DEADLINE_REFRESH_PERIOD: Duration = Duration::from_secs(600); + +pub(super) fn doe_arm_delay(base_delay: Duration, min_age: Duration) -> Duration { + base_delay + .saturating_add(min_age) + .saturating_add(DOE_DEADLINE_REFRESH_PERIOD) +} #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) struct StreamerGenerationId(u64); @@ -74,6 +80,7 @@ struct InFlightAppend { struct DbSubmitAppendOptions { retention: RetentionPolicy, + doe_deadline: Option, fencing_token: Option, trim_point: Option>, } @@ -203,14 +210,10 @@ pub(super) struct Spawner { pub generation_id: StreamerGenerationId, pub db: slatedb::Db, pub stream_id: StreamId, - /// Database commit sequence that created the stream's ID mapping. - /// Stable across streamer restarts; changes when the stream is recreated. - pub stream_creation_seq: u64, pub config: StreamConfig, - pub config_seq: u64, pub cipher: Option, pub tail_pos: StreamPosition, - pub last_tail_write_timestamp: TimestampSecs, + pub last_tail_write_timestamp: kv::timestamp::TimestampSecs, pub fencing_token: FencingToken, pub trim_point: RangeTo, pub append_inflight_bytes_sema: Arc, @@ -227,9 +230,7 @@ impl Spawner { generation_id, db, stream_id, - stream_creation_seq, config, - config_seq, cipher, tail_pos, last_tail_write_timestamp, @@ -245,10 +246,8 @@ impl Spawner { let streamer = Streamer { db, stream_id, - stream_creation_seq, msg_tx: msg_tx.clone(), config, - config_seq, last_tail_write_timestamp, fencing_token: CommandState { state: fencing_token, @@ -258,6 +257,7 @@ impl Spawner { state: trim_point, applied_point: ..tail_pos.seq_num, }, + last_doe_deadline_at: None, db_writes_pending: VecDeque::new(), db_durability_subscription: 0, inflight_appends: VecDeque::new(), @@ -306,13 +306,12 @@ impl CommandState { struct Streamer { db: slatedb::Db, stream_id: StreamId, - stream_creation_seq: u64, msg_tx: mpsc::UnboundedSender, config: StreamConfig, - config_seq: u64, - last_tail_write_timestamp: TimestampSecs, + last_tail_write_timestamp: kv::timestamp::TimestampSecs, fencing_token: CommandState, trim_point: CommandState>, + last_doe_deadline_at: Option, db_writes_pending: VecDeque>>, db_durability_subscription: u64, inflight_appends: VecDeque, @@ -401,10 +400,7 @@ impl Streamer { return; }; let sequenced_records = if self.trim_point.state.end == SeqNum::MAX { - Err(AppendErrorInternal::StreamDeletionPending { - // The terminal trim must be durable before reporting deletion pending. - durability_dependency: self.trim_point.applied_point, - }) + Err(StreamDeletionPendingError.into()) } else { self.sequence_records(input) }; @@ -426,6 +422,7 @@ impl Streamer { let seq_num_range = first_pos.seq_num..next_pos.seq_num; let opts = DbSubmitAppendOptions { retention: self.config.retention_policy, + doe_deadline: self.doe_deadline_maybe(), fencing_token: self .fencing_token .is_applied_in(&seq_num_range) @@ -440,7 +437,7 @@ impl Streamer { .boxed(), ); self.pending_appends.accept(ticket, first_pos..next_pos); - self.last_tail_write_timestamp = TimestampSecs::now(); + self.last_tail_write_timestamp = kv::timestamp::TimestampSecs::now(); } Err(e) => { self.pending_appends.reject(ticket, e, self.stable_pos); @@ -455,41 +452,30 @@ impl Streamer { ) { match condition { TerminalTrimCondition::Always => { - self.ensure_terminal_trim(reply_tx); + self.append_terminal_trim(reply_tx); } - TerminalTrimCondition::DeleteOnEmpty { - expected_stream_creation_seq, - expected_config_seq, - } => { - if self.stream_creation_seq != expected_stream_creation_seq { - let _ = reply_tx.send(Ok(TerminalTrimOutcome::Obsolete)); - } else if self.trim_point.state.end == SeqNum::MAX { - self.ensure_terminal_trim(reply_tx); - } else if self.config_seq != expected_config_seq { - // The worker may have observed a configuration commit before - // its notification reached this actor (or vice versa). Do not - // defer using an age that may have just been decreased. - let _ = reply_tx.send(Ok(TerminalTrimOutcome::RetryAt(TimestampSecs::after( - doe::RETRY_INTERVAL, - )))); - } else if self.config.delete_on_empty.min_age().is_none() { - let _ = reply_tx.send(Ok(TerminalTrimOutcome::Obsolete)); - } else { - let db = self.db.clone(); - let stream_id = self.stream_id; - let stable_pos_snapshot = self.stable_pos; - let config_seq_snapshot = self.config_seq; - let msg_tx = self.msg_tx.clone(); - tokio::spawn(async move { - let records = stream_record_presence(&db, stream_id).await; - let _ = msg_tx.send(Message::DeleteOnEmptyCheckResult { - stable_pos_snapshot, - config_seq_snapshot, - records, - reply_tx, - }); - }); + TerminalTrimCondition::DeleteOnEmpty { last_write_cutoff } => { + if self.last_tail_write_timestamp > last_write_cutoff + || self.next_assignable_pos().seq_num != self.stable_pos.seq_num + || self.config.delete_on_empty.min_age().is_none() + { + let _ = reply_tx.send(Ok(TerminalTrimOutcome::Ineligible)); + return; } + + let db = self.db.clone(); + let stream_id = self.stream_id; + let stable_pos_snapshot = self.stable_pos; + let msg_tx = self.msg_tx.clone(); + tokio::spawn(async move { + let has_records = stream_has_records(&db, stream_id).await; + let _ = msg_tx.send(Message::DeleteOnEmptyCheckResult { + stable_pos_snapshot, + last_write_cutoff, + has_records, + reply_tx, + }); + }); } } } @@ -497,66 +483,34 @@ impl Streamer { fn handle_doe_check_result( &mut self, stable_pos_snapshot: StreamPosition, - config_seq_snapshot: u64, - records: Result, + last_write_cutoff: kv::timestamp::TimestampSecs, + has_records: Result, reply_tx: oneshot::Sender>, ) { - let records = match records { - Ok(records) => records, + match has_records { + Ok(true) => { + let _ = reply_tx.send(Ok(TerminalTrimOutcome::Ineligible)); + } + Ok(false) => { + if self.trim_point.state.end == SeqNum::MAX { + let _ = reply_tx.send(Ok(TerminalTrimOutcome::DeletionPending)); + } else if self.stable_pos != stable_pos_snapshot + || self.next_assignable_pos() != stable_pos_snapshot + || self.last_tail_write_timestamp > last_write_cutoff + || self.config.delete_on_empty.min_age().is_none() + { + let _ = reply_tx.send(Ok(TerminalTrimOutcome::Ineligible)); + } else { + self.append_terminal_trim(reply_tx); + } + } Err(err) => { let _ = reply_tx.send(Err(err.into())); - return; } - }; - if self.trim_point.state.end == SeqNum::MAX { - self.ensure_terminal_trim(reply_tx); - return; } - if self.config_seq != config_seq_snapshot { - let _ = reply_tx.send(Ok(TerminalTrimOutcome::RetryAt(TimestampSecs::after( - doe::RETRY_INTERVAL, - )))); - return; - } - let Some(min_age) = self.config.delete_on_empty.min_age() else { - let _ = reply_tx.send(Ok(TerminalTrimOutcome::Obsolete)); - return; - }; - let outcome = match records { - // Appends cannot remove an observed record, so these bounds remain - // useful even when the tail advances. The scheduler's revision check - // protects against a trim removing the record before we finish. - RecordPresence::ExpiresAt(at) => self.doe_retry_at(at), - RecordPresence::Unbounded => TerminalTrimOutcome::Parked, - RecordPresence::Empty => { - let old_enough = TimestampSecs::now() - .checked_sub_duration(min_age) - .is_some_and(|cutoff| self.last_tail_write_timestamp <= cutoff); - if self.stable_pos == stable_pos_snapshot - && self.next_assignable_pos() == stable_pos_snapshot - && old_enough - { - self.ensure_terminal_trim(reply_tx); - return; - } - self.doe_retry_at(TimestampSecs::ZERO) - } - }; - let _ = reply_tx.send(Ok(outcome)); } - fn doe_retry_at(&self, earliest_empty: TimestampSecs) -> TerminalTrimOutcome { - let age_at = self - .last_tail_write_timestamp - .saturating_add_duration(self.config.delete_on_empty.min_age().unwrap_or_default()); - TerminalTrimOutcome::RetryAt( - TimestampSecs::after(doe::RETRY_INTERVAL) - .max(age_at) - .max(earliest_empty), - ) - } - - fn ensure_terminal_trim( + fn append_terminal_trim( &mut self, reply_tx: oneshot::Sender>, ) { @@ -570,7 +524,7 @@ impl Streamer { tokio::spawn(async move { let result = match append_reply_rx.await { Ok(Ok(_)) => Ok(TerminalTrimOutcome::DeletionPending), - Ok(Err(AppendErrorInternal::StreamDeletionPending { .. })) => { + Ok(Err(AppendErrorInternal::StreamDeletionPending(_))) => { Ok(TerminalTrimOutcome::DeletionPending) } Ok(Err(AppendErrorInternal::Storage(e))) => Err(DeleteStreamError::Storage(e)), @@ -589,12 +543,31 @@ impl Streamer { Ok(Err(AppendErrorInternal::MaxSeqNum(_))) => { unreachable!("terminal append is plaintext command record") } - Err(_) => Err(RequestDroppedError.into()), + Err(_) => Err(DeleteStreamError::StreamerMissingInActionError( + StreamerMissingInActionError, + )), }; let _ = reply_tx.send(result); }); } + fn doe_deadline_maybe(&mut self) -> Option { + let retention_age = self.config.retention_policy.age()?; + let min_age = self.config.delete_on_empty.min_age()?; + let now = Instant::now(); + if self + .last_doe_deadline_at + .is_none_or(|t| now.duration_since(t) >= DOE_DEADLINE_REFRESH_PERIOD) + { + self.last_doe_deadline_at = Some(now); + let deadline = + kv::timestamp::TimestampSecs::after(doe_arm_delay(retention_age, min_age)); + Some(kv::stream_doe_deadline::Entry { deadline, min_age }) + } else { + None + } + } + fn subscribe_durability(&mut self) { if let Some(inflight_append) = self .inflight_appends @@ -686,14 +659,14 @@ impl Streamer { } Message::DeleteOnEmptyCheckResult { stable_pos_snapshot, - config_seq_snapshot, - records, + last_write_cutoff, + has_records, reply_tx, } => { self.handle_doe_check_result( stable_pos_snapshot, - config_seq_snapshot, - records, + last_write_cutoff, + has_records, reply_tx, ); } @@ -711,11 +684,8 @@ impl Streamer { Message::CheckTail { reply_tx } => { let _ = reply_tx.send(self.stable_pos); } - Message::Reconfigure { seq, config } => { - if seq > self.config_seq { - self.config = config; - self.config_seq = seq; - } + Message::Reconfigure { config } => { + self.config = config; } Message::DurabilityStatus(status) => { match status { @@ -736,12 +706,7 @@ impl Streamer { } } _ = dormancy.as_mut() => { - // Cancelled requests can still have writes become durable. Keep - // their assigned positions until a new streamer can recover them. - if self.db_writes_pending.is_empty() - && self.inflight_appends.is_empty() - && self.lease_state.close_if_idle() - { + if self.lease_state.close_if_idle() { break; } } @@ -763,8 +728,8 @@ enum Message { }, DeleteOnEmptyCheckResult { stable_pos_snapshot: StreamPosition, - config_seq_snapshot: u64, - records: Result, + last_write_cutoff: kv::timestamp::TimestampSecs, + has_records: Result, reply_tx: oneshot::Sender>, }, Follow { @@ -777,7 +742,6 @@ enum Message { reply_tx: oneshot::Sender, }, Reconfigure { - seq: u64, config: StreamConfig, }, DurabilityStatus(Result), @@ -786,19 +750,14 @@ enum Message { pub(super) enum TerminalTrimCondition { Always, DeleteOnEmpty { - expected_stream_creation_seq: u64, - expected_config_seq: u64, + last_write_cutoff: kv::timestamp::TimestampSecs, }, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum TerminalTrimOutcome { - /// Deletion is durably pending. DeletionPending, - RetryAt(TimestampSecs), - Parked, - /// DOE is disabled or this request belongs to an earlier incarnation. - Obsolete, + Ineligible, } #[derive(Debug, Clone)] @@ -878,10 +837,8 @@ impl StreamerClient { }) } - pub(super) fn advise_reconfig(&self, seq: u64, config: StreamConfig) -> bool { - self.msg_tx - .send(Message::Reconfigure { seq, config }) - .is_ok() + pub(super) fn advise_reconfig(&self, config: StreamConfig) -> bool { + self.msg_tx.send(Message::Reconfigure { config }).is_ok() } async fn terminal_trim( @@ -897,7 +854,9 @@ impl StreamerClient { .map_err(|_| { DeleteStreamError::StreamerMissingInActionError(StreamerMissingInActionError) })?; - reply_rx.await.map_err(|_| RequestDroppedError)? + reply_rx.await.map_err(|_| { + DeleteStreamError::StreamerMissingInActionError(StreamerMissingInActionError) + })? } } @@ -986,17 +945,7 @@ pub fn next_pos(records: &[Metered]) -> StreamPosition { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum RecordPresence { - Empty, - ExpiresAt(TimestampSecs), - Unbounded, -} - -async fn stream_record_presence( - db: &slatedb::Db, - stream_id: StreamId, -) -> Result { +async fn stream_has_records(db: &slatedb::Db, stream_id: StreamId) -> Result { let prefix = kv::stream_record_timestamp::ser_key_prefix(stream_id); let scan_opts = ScanOptions::default().with_order(IterationOrder::Descending); let mut it = db.scan_prefix_with_options(prefix, .., &scan_opts).await?; @@ -1005,20 +954,11 @@ async fn stream_record_presence( .map(|duration| i64::try_from(duration.as_millis()).unwrap_or(i64::MAX)) .unwrap_or(0); while let Some(kv) = it.next().await? { - match kv.expire_ts { - None => return Ok(RecordPresence::Unbounded), - Some(expire_ts) if expire_ts > now_millis => { - // One live record bounds when the stream can become empty. - // Use its stored TTL, since retention changes are not retroactive. - // Round up so the check does not run just before expiration. - return Ok(RecordPresence::ExpiresAt(TimestampSecs::from_millis( - expire_ts.saturating_add(999), - ))); - } - _ => (), + if kv.expire_ts.is_none_or(|expire_ts| expire_ts > now_millis) { + return Ok(true); } } - Ok(RecordPresence::Empty) + Ok(false) } fn sequenced_records( @@ -1073,46 +1013,56 @@ async fn db_submit_append( records: Vec>, DbSubmitAppendOptions { retention, + doe_deadline, fencing_token, trim_point, }: DbSubmitAppendOptions, ) -> Result { let ttl = match retention { - RetentionPolicy::Age(age) => Ttl::ExpireAfterMillis(age.as_millis() as u64), + RetentionPolicy::Age(age) => Ttl::ExpireAfter(age.as_millis() as u64), RetentionPolicy::Infinite() => Ttl::NoExpiry, }; let ttl_put_opts = PutOptions { ttl }; let mut wb = WriteBatch::new(); for (position, record) in records.iter().map(|msr| msr.parts()) { - wb.put_bytes_with_options( + wb.put_with_options( kv::stream_record_data::ser_key(stream_id, position), kv::stream_record_data::ser_value(record), &ttl_put_opts, ); - wb.put_bytes_with_options( + wb.put_with_options( kv::stream_record_timestamp::ser_key(stream_id, position), kv::stream_record_timestamp::ser_value(), &ttl_put_opts, ); } if let Some(fencing_token) = fencing_token { - wb.put_bytes( + wb.put( kv::stream_fencing_token::ser_key(stream_id), kv::stream_fencing_token::ser_value(&fencing_token), ); } if let Some(trim_point) = trim_point.and_then(|tp| NonZeroSeqNum::new(tp.end)) { - wb.put_bytes( + wb.put( kv::stream_trim_point::ser_key(stream_id), kv::stream_trim_point::ser_value(..trim_point), ); } - wb.put_bytes( + if let Some(doe_deadline) = doe_deadline { + wb.put( + kv::stream_doe_deadline::ser_key(doe_deadline.deadline, stream_id), + kv::stream_doe_deadline::ser_value(doe_deadline.min_age), + ); + } + wb.put( kv::stream_tail_position::ser_key(stream_id), kv::stream_tail_position::ser_value(next_pos(&records)), ); - // The durability notifier tracks this sequence and acknowledges the append after flush. - let write_handle = db.write(wb).await?; + let write_opts = WriteOptions { + await_durable: false, + ..Default::default() + }; + let write_handle = db.write_with_options(wb, &write_opts).await?; Ok(InFlightAppend { db_seq: write_handle.seqnum(), records, @@ -1459,27 +1409,9 @@ mod tests { } } - async fn make_pending_append_durable(streamer: &mut Streamer) { - let submitted = streamer - .db_writes_pending - .pop_front() - .unwrap() - .await - .unwrap(); - let seq = submitted.db_seq; - streamer.inflight_appends.push_back(submitted); - streamer.db.flush().await.unwrap(); - streamer.on_db_durable_seq_advanced(seq); - } - async fn test_streamer() -> Streamer { - test_streamer_with_settings(Default::default()).await - } - - async fn test_streamer_with_settings(settings: slatedb::config::Settings) -> Streamer { let object_store = Arc::new(InMemory::new()); let db = slatedb::Db::builder("/test", object_store) - .with_settings(settings) .build() .await .expect("db"); @@ -1489,11 +1421,9 @@ mod tests { Streamer { db: db.clone(), stream_id: [3u8; StreamId::LEN].into(), - stream_creation_seq: 0, msg_tx, config: StreamConfig::default(), - config_seq: 0, - last_tail_write_timestamp: TimestampSecs::ZERO, + last_tail_write_timestamp: kv::timestamp::TimestampSecs::ZERO, fencing_token: CommandState { state: FencingToken::default(), applied_point: ..SeqNum::MIN, @@ -1502,6 +1432,7 @@ mod tests { state: ..SeqNum::MIN, applied_point: ..SeqNum::MIN, }, + last_doe_deadline_at: None, db_writes_pending: VecDeque::new(), db_durability_subscription: 0, inflight_appends: VecDeque::new(), @@ -1514,62 +1445,6 @@ mod tests { } } - #[tokio::test] - async fn stale_config_notifications_cannot_restore_old_retention() { - let mut streamer = test_streamer().await; - let db = streamer.db.clone(); - let stream_id = streamer.stream_id; - let (msg_tx, msg_rx) = mpsc::unbounded_channel(); - streamer.msg_tx = msg_tx.clone(); - streamer.config.retention_policy = RetentionPolicy::Age(Duration::from_secs(1)); - let task = tokio::spawn(streamer.run(msg_rx)); - msg_tx - .send(Message::Reconfigure { - seq: 20, - config: StreamConfig { - retention_policy: RetentionPolicy::Infinite(), - ..Default::default() - }, - }) - .unwrap(); - let old = StreamConfig { - retention_policy: RetentionPolicy::Age(Duration::from_secs(1)), - ..Default::default() - }; - msg_tx - .send(Message::Reconfigure { - seq: 10, - config: old, - }) - .unwrap(); - let (reply_tx, reply_rx) = oneshot::channel(); - msg_tx - .send(Message::Append { - input: append_input(b"must not expire"), - session: None, - reply_tx, - append_type: AppendType::Regular, - }) - .unwrap(); - let ack = tokio::time::timeout(Duration::from_secs(5), reply_rx) - .await - .unwrap() - .unwrap() - .unwrap(); - let entry = db - .get_key_value(kv::stream_record_data::ser_key(stream_id, ack.start)) - .await - .unwrap() - .unwrap(); - assert!( - entry.expire_ts.is_none(), - "late config notification restored stale retention" - ); - task.abort(); - assert!(task.await.unwrap_err().is_cancelled()); - db.close().await.unwrap(); - } - #[test] fn lease_state_closes_when_idle_and_rejects_new_leases() { let (streamer_lease_state, client_lease_state) = StreamerLeaseState::new(); @@ -1614,304 +1489,60 @@ mod tests { } #[tokio::test] - async fn terminal_trim_and_rejections_wait_for_durability() { - let mut streamer = test_streamer_with_settings(slatedb::config::Settings { - flush_interval: None, - ..Default::default() - }) - .await; - let mut replies = Vec::new(); - for _ in 0..2 { - let (tx, rx) = oneshot::channel(); - streamer.handle_terminal_trim(TerminalTrimCondition::Always, tx); - replies.push(rx); - } - // An empty-stream check can also finish after another deletion request starts. - let (tx, rx) = oneshot::channel(); - streamer.handle_doe_check_result( - StreamPosition::MIN, - streamer.config_seq, - Ok(RecordPresence::Empty), - tx, - ); - replies.push(rx); + async fn append_during_terminal_trim_returns_stream_deletion_pending() { + let mut streamer = test_streamer().await; + streamer.trim_point = CommandState { + state: ..SeqNum::MAX, + applied_point: ..1, + }; + let (msg_tx, msg_rx) = mpsc::unbounded_channel(); + let run_handle = tokio::spawn(streamer.run(msg_rx)); - let (tx, mut append_reply) = oneshot::channel(); - streamer.handle_append(append_input(b"late"), None, tx, AppendType::Regular); - assert_eq!(streamer.db_writes_pending.len(), 1); - tokio::task::yield_now().await; - assert!(matches!( - append_reply.try_recv(), - Err(oneshot::error::TryRecvError::Empty) - )); - for reply in &mut replies { - assert!( - matches!(reply.try_recv(), Err(oneshot::error::TryRecvError::Empty)), - "stream deletion must wait even when the original trim has not been submitted" - ); - } + let (reply_tx, reply_rx) = oneshot::channel(); + msg_tx + .send(Message::Append { + input: append_input(b"late"), + session: None, + reply_tx, + append_type: AppendType::Regular, + }) + .expect("streamer should accept append message"); - let submitted = streamer - .db_writes_pending - .pop_front() - .unwrap() + let err = reply_rx .await - .unwrap(); - let db_seq = submitted.db_seq; - streamer.inflight_appends.push_back(submitted); - tokio::task::yield_now().await; + .expect("streamer should reply") + .expect_err("append should be rejected"); + let AppendErrorInternal::StreamDeletionPending(_) = err else { + panic!("expected stream deletion pending"); + }; + + run_handle.abort(); + } + + #[tokio::test] + async fn delete_on_empty_terminal_trim_skips_pending_append() { + let mut streamer = test_streamer().await; + let (append_tx, mut append_rx) = oneshot::channel(); + streamer.handle_append(append_input(b"live"), None, append_tx, AppendType::Regular); + assert_eq!(streamer.db_writes_pending.len(), 1); assert!(matches!( - append_reply.try_recv(), - Err(oneshot::error::TryRecvError::Empty) + append_rx.try_recv(), + Err(tokio::sync::oneshot::error::TryRecvError::Empty) )); - for reply in &mut replies { - assert!( - matches!(reply.try_recv(), Err(oneshot::error::TryRecvError::Empty)), - "a committed but unflushed trim must not acknowledge stream deletion" - ); - } - streamer.db.flush().await.unwrap(); - streamer.on_db_durable_seq_advanced(db_seq); - assert!(matches!( - append_reply.await.unwrap(), - Err(AppendErrorInternal::StreamDeletionPending { .. }) - )); - for reply in replies { - assert_eq!( - reply.await.unwrap().unwrap(), - TerminalTrimOutcome::DeletionPending - ); - } - streamer.db.close().await.unwrap(); - } - #[rstest::rstest] - #[case::pending_before_scan(false, false)] - #[case::pending_after_scan(true, false)] - #[case::durable_after_scan(true, true)] - #[tokio::test] - async fn delete_on_empty_uses_nonempty_observation_despite_appends( - #[case] append_after_scan: bool, - #[case] durable: bool, - #[values(false, true)] infinite: bool, - ) { - let mut streamer = test_streamer().await; - streamer.config.delete_on_empty.min_age = Duration::from_secs(60); - streamer.config.retention_policy = if infinite { - RetentionPolicy::Infinite() - } else { - RetentionPolicy::Age(Duration::from_secs(3600)) - }; - let (seed_tx, seed_rx) = oneshot::channel(); - streamer.handle_append( - append_input(b"observed record"), - None, - seed_tx, - AppendType::Regular, - ); - make_pending_append_durable(&mut streamer).await; - seed_rx.await.unwrap().unwrap(); - - let (msg_tx, mut msg_rx) = mpsc::unbounded_channel(); - streamer.msg_tx = msg_tx; - let (append_tx, append_rx) = oneshot::channel(); - // Keep the sender until the selected point in the asynchronous check. - let mut append_tx = Some(append_tx); - if !append_after_scan { - streamer.handle_append( - append_input(b"pending before scan"), - None, - append_tx.take().unwrap(), - AppendType::Regular, - ); - } - let (reply_tx, reply_rx) = oneshot::channel(); + let (trim_tx, trim_rx) = oneshot::channel(); streamer.handle_terminal_trim( TerminalTrimCondition::DeleteOnEmpty { - expected_stream_creation_seq: streamer.stream_creation_seq, - expected_config_seq: streamer.config_seq, + last_write_cutoff: kv::timestamp::TimestampSecs::MAX, }, - reply_tx, + trim_tx, ); - let Message::DeleteOnEmptyCheckResult { - stable_pos_snapshot, - config_seq_snapshot, - records, - reply_tx, - } = msg_rx.recv().await.unwrap() - else { - panic!("expected record check even with a pending append"); - }; - if append_after_scan { - streamer.handle_append( - append_input(b"arrived after scan"), - None, - append_tx.take().unwrap(), - AppendType::Regular, - ); - } - if durable { - make_pending_append_durable(&mut streamer).await; - append_rx.await.unwrap().unwrap(); - } - streamer.handle_doe_check_result( - stable_pos_snapshot, - config_seq_snapshot, - records, - reply_tx, - ); - assert_eq!(streamer.db_writes_pending.len(), usize::from(!durable)); - assert_ne!(streamer.trim_point.state.end, SeqNum::MAX); - let outcome = reply_rx.await.unwrap().unwrap(); - if infinite { - assert_eq!(outcome, TerminalTrimOutcome::Parked); - } else { - assert!(matches!( - outcome, - TerminalTrimOutcome::RetryAt(at) - if at > TimestampSecs::after(Duration::from_secs(3500)) - )); - } - streamer.db.close().await.unwrap(); - } - #[rstest::rstest] - #[case::pending_before_scan(false, false)] - #[case::pending_after_scan(true, false)] - #[case::durable_after_scan(true, true)] - #[tokio::test] - async fn delete_on_empty_rechecks_appends_after_empty_scan( - #[case] append_after_scan: bool, - #[case] durable: bool, - ) { - let mut streamer = test_streamer().await; - streamer.config.delete_on_empty.min_age = Duration::from_secs(1); - let (msg_tx, mut msg_rx) = mpsc::unbounded_channel(); - streamer.msg_tx = msg_tx; - let (append_tx, append_rx) = oneshot::channel(); - let mut append_tx = Some(append_tx); - if !append_after_scan { - streamer.handle_append( - append_input(b"pending before scan"), - None, - append_tx.take().unwrap(), - AppendType::Regular, - ); - } - let (reply_tx, reply_rx) = oneshot::channel(); - streamer.handle_terminal_trim( - TerminalTrimCondition::DeleteOnEmpty { - expected_stream_creation_seq: 0, - expected_config_seq: 0, - }, - reply_tx, + assert_eq!( + trim_rx.await.expect("terminal trim reply").unwrap(), + TerminalTrimOutcome::Ineligible ); - let Message::DeleteOnEmptyCheckResult { - stable_pos_snapshot, - config_seq_snapshot, - records, - reply_tx, - } = msg_rx.recv().await.unwrap() - else { - panic!("expected empty-stream check result"); - }; - assert_eq!(records.as_ref().unwrap(), &RecordPresence::Empty); - if append_after_scan { - streamer.handle_append( - append_input(b"arrived during scan"), - None, - append_tx.take().unwrap(), - AppendType::Regular, - ); - } - if durable { - make_pending_append_durable(&mut streamer).await; - append_rx.await.unwrap().unwrap(); - } - // Let the minimum age elapse so it cannot mask a missing tail check. - tokio::time::sleep(Duration::from_millis(1100)).await; - streamer.handle_doe_check_result( - stable_pos_snapshot, - config_seq_snapshot, - records, - reply_tx, - ); - assert_eq!(streamer.db_writes_pending.len(), usize::from(!durable)); - assert_ne!(streamer.trim_point.state.end, SeqNum::MAX); - assert!(matches!( - reply_rx.await.unwrap().unwrap(), - TerminalTrimOutcome::RetryAt(_) - )); - streamer.db.close().await.unwrap(); - } - - #[rstest::rstest] - #[case::actor_behind(0, 1)] - #[case::worker_behind(1, 0)] - #[tokio::test] - async fn delete_on_empty_bounds_retry_for_stale_config( - #[case] actor_seq: u64, - #[case] worker_seq: u64, - ) { - let mut streamer = test_streamer().await; - streamer.config.delete_on_empty.min_age = Duration::from_secs(365 * 24 * 3600); - streamer.last_tail_write_timestamp = TimestampSecs::now(); - streamer.config_seq = actor_seq; - let earliest_retry = TimestampSecs::after(doe::RETRY_INTERVAL); - let (reply_tx, reply_rx) = oneshot::channel(); - streamer.handle_terminal_trim( - TerminalTrimCondition::DeleteOnEmpty { - expected_stream_creation_seq: 0, - expected_config_seq: worker_seq, - }, - reply_tx, - ); - let TerminalTrimOutcome::RetryAt(at) = reply_rx.await.unwrap().unwrap() else { - panic!("expected a bounded retry for mismatched configuration"); - }; - assert!(at >= earliest_retry && at <= TimestampSecs::after(doe::RETRY_INTERVAL)); - assert!(streamer.db_writes_pending.is_empty()); - streamer.db.close().await.unwrap(); - } - - #[tokio::test] - async fn delete_on_empty_bounds_retry_when_config_changes_during_scan() { - let mut streamer = test_streamer().await; - streamer.config.delete_on_empty.min_age = Duration::from_secs(365 * 24 * 3600); - streamer.last_tail_write_timestamp = TimestampSecs::now(); - let (msg_tx, mut msg_rx) = mpsc::unbounded_channel(); - streamer.msg_tx = msg_tx; - let (reply_tx, reply_rx) = oneshot::channel(); - streamer.handle_terminal_trim( - TerminalTrimCondition::DeleteOnEmpty { - expected_stream_creation_seq: 0, - expected_config_seq: 0, - }, - reply_tx, - ); - let Message::DeleteOnEmptyCheckResult { - stable_pos_snapshot, - config_seq_snapshot, - records, - reply_tx, - } = msg_rx.recv().await.unwrap() - else { - panic!("expected record check"); - }; - streamer.config_seq += 1; - let earliest_retry = TimestampSecs::after(doe::RETRY_INTERVAL); - streamer.handle_doe_check_result( - stable_pos_snapshot, - config_seq_snapshot, - records, - reply_tx, - ); - let TerminalTrimOutcome::RetryAt(at) = reply_rx.await.unwrap().unwrap() else { - panic!("expected a bounded retry for mismatched configuration"); - }; - assert!(at >= earliest_retry && at <= TimestampSecs::after(doe::RETRY_INTERVAL)); - assert!(streamer.db_writes_pending.is_empty()); - streamer.db.close().await.unwrap(); + assert_eq!(streamer.db_writes_pending.len(), 1); } #[tokio::test] @@ -2044,50 +1675,4 @@ mod tests { assert_eq!(streamer.stable_pos.seq_num, 4); assert!(streamer.inflight_appends.is_empty()); } - - #[tokio::test(start_paused = true)] - async fn cancelled_append_delays_dormancy_until_writes_are_durable() { - let mut streamer = test_streamer_with_settings(slatedb::config::Settings { - flush_interval: None, - ..Default::default() - }) - .await; - let db = streamer.db.clone(); - let (msg_tx, msg_rx) = mpsc::unbounded_channel(); - streamer.msg_tx = msg_tx; - - let (reply_tx, reply_rx) = oneshot::channel(); - streamer.handle_append( - append_input(b"cancelled"), - None, - reply_tx, - AppendType::Regular, - ); - let task = tokio::spawn(streamer.run(msg_rx)); - tokio::time::timeout(Duration::from_secs(5), async { - while db.snapshot().await.unwrap().seq() == 0 { - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - drop(reply_rx); - tokio::time::sleep(DORMANT_TIMEOUT + Duration::from_secs(1)).await; - assert_eq!( - db.status().durable_seq, - 0, - "the write must still be unflushed" - ); - assert!( - !task.is_finished(), - "dormancy abandoned an unflushed append" - ); - - db.flush().await.unwrap(); - tokio::time::timeout(DORMANT_TIMEOUT + Duration::from_secs(1), task) - .await - .expect("durable writes should allow normal dormancy") - .unwrap(); - db.close().await.unwrap(); - } } diff --git a/lite/src/backend/streams.rs b/lite/src/backend/streams.rs index b74ca6f3..b59d8032 100644 --- a/lite/src/backend/streams.rs +++ b/lite/src/backend/streams.rs @@ -1,23 +1,22 @@ use s2_common::{ basin::BasinName, - config::{DeleteOnEmptyConfig, OptionalStreamConfig, StreamConfig, StreamReconfiguration}, - record::{NonZeroSeqNum, StreamPosition}, + config::{OptionalStreamConfig, StreamConfig, StreamReconfiguration}, + record::StreamPosition, resources::{Page, ProvisionMode, ProvisionResult, RequestToken}, stream::{ListStreamsRequest, StreamInfo, StreamName}, }; use s2_storage::bash::Bash; use slatedb::{ - DbTransaction, IsolationLevel, + IsolationLevel, config::{DurabilityLevel, ScanOptions}, }; use time::OffsetDateTime; use tracing::instrument; use super::{ - Backend, doe, resolve_stream_config, - store::{db_txn_commit_durable, db_txn_get, db_txn_get_with}, - streamer::{TerminalTrimCondition, TerminalTrimOutcome}, - timestamp::TimestampSecs, + Backend, + store::db_txn_get, + streamer::{TerminalTrimCondition, TerminalTrimOutcome, doe_arm_delay}, }; use crate::{ backend::{ @@ -107,26 +106,22 @@ impl Backend { // Existence is decided from a Memory-level read; capture the row's // commit seq so exists-outcomes can await durability (see fn doc). - let (existing_meta, existing_seq) = db_txn_get_with(&txn, &stream_meta_key, |entry| { - Ok((kv::stream_meta::deser_value(entry.value)?, entry.seq)) - }) - .await? - .unzip(); - if existing_meta - .as_ref() - .is_some_and(|meta| meta.deleted_at.is_some()) - || has_terminal_trim(&txn, StreamId::new(&basin, &stream)).await? + let existing_entry = txn.get_key_value(&stream_meta_key).await?; + let existing_seq = existing_entry.as_ref().map(|kv| kv.seq); + let existing_meta = existing_entry + .map(|kv| kv::stream_meta::deser_value(kv.value)) + .transpose() + .map_err(StorageError::from)?; + if let Some(existing_meta) = &existing_meta + && existing_meta.deleted_at.is_some() { return Err(ProvisionStreamError::StreamDeletionPending( StreamDeletionPendingError, )); } - let prior_doe = existing_meta - .as_ref() - .map(|meta| meta.config.delete_on_empty); let basin_defaults = basin_meta.config.default_stream_config; - let outcome = match (existing_meta, mode) { + let (outcome, prior_doe_min_age) = match (existing_meta, mode) { (Some(existing), ProvisionMode::CreateOnly { request_token }) => { let new_creation_idempotency_key = request_token .as_ref() @@ -149,7 +144,7 @@ impl Backend { return result; } (Some(existing), ProvisionMode::Ensure) => { - let desired_config = resolve_stream_config(config, basin_defaults); + let desired_config = config.merge(basin_defaults); let config_unchanged = existing.config == desired_config; let meta = kv::stream_meta::StreamMeta { config: desired_config, @@ -158,33 +153,40 @@ impl Backend { deleted_at: None, creation_idempotency_key: existing.creation_idempotency_key, }; - if config_unchanged { - ProvisionResult::Noop(meta) - } else { - ProvisionResult::Updated(meta) - } + ( + if config_unchanged { + ProvisionResult::Noop(meta) + } else { + ProvisionResult::Updated(meta) + }, + existing.config.delete_on_empty.min_age(), + ) } (None, ProvisionMode::CreateOnly { request_token }) => { let new_creation_idempotency_key = request_token .as_ref() .map(|req_token| creation_idempotency_key(req_token, &config)); - ProvisionResult::Created(kv::stream_meta::StreamMeta { - config: resolve_stream_config(config, basin_defaults), - cipher: basin_meta.config.stream_cipher, - created_at: OffsetDateTime::now_utc(), - deleted_at: None, - creation_idempotency_key: new_creation_idempotency_key, - }) + ( + ProvisionResult::Created(kv::stream_meta::StreamMeta { + config: config.merge(basin_defaults), + cipher: basin_meta.config.stream_cipher, + created_at: OffsetDateTime::now_utc(), + deleted_at: None, + creation_idempotency_key: new_creation_idempotency_key, + }), + None, + ) } - (None, ProvisionMode::Ensure) => { + (None, ProvisionMode::Ensure) => ( ProvisionResult::Created(kv::stream_meta::StreamMeta { - config: resolve_stream_config(config, basin_defaults), + config: config.merge(basin_defaults), cipher: basin_meta.config.stream_cipher, created_at: OffsetDateTime::now_utc(), deleted_at: None, creation_idempotency_key: None, - }) - } + }), + None, + ), }; if matches!(&outcome, ProvisionResult::Noop(_)) { @@ -209,14 +211,28 @@ impl Backend { )?; } - self.commit_stream_config( - txn, - basin.clone(), - stream.clone(), - meta.config.clone(), - prior_doe, - ) - .await?; + if let Some(min_age) = meta.config.delete_on_empty.min_age() + && (matches!(&outcome, ProvisionResult::Created(_)) || prior_doe_min_age.is_none()) + { + txn.put( + kv::stream_doe_deadline::ser_key( + kv::timestamp::TimestampSecs::after(doe_arm_delay( + meta.config.retention_policy.age().unwrap_or_default(), + min_age, + )), + stream_id, + ), + kv::stream_doe_deadline::ser_value(min_age), + )?; + } + + txn.commit().await?; + } + + if let ProvisionResult::Updated(meta) = &outcome + && let Some(client) = self.streamer_client_if_active(&basin, &stream) + { + client.advise_reconfig(meta.config.clone()); } Ok(outcome.map(|meta| StreamInfo { @@ -227,6 +243,17 @@ impl Backend { })) } + pub(super) async fn stream_id_mapping( + &self, + stream_id: StreamId, + ) -> Result, StorageError> { + self.db_get( + kv::stream_id_mapping::ser_key(stream_id), + kv::stream_id_mapping::deser_value, + ) + .await + } + pub async fn get_stream_config( &self, basin: BasinName, @@ -278,60 +305,41 @@ impl Backend { stream: stream.clone(), })?; - if meta.deleted_at.is_some() - || has_terminal_trim(&txn, StreamId::new(&basin, &stream)).await? - { + if meta.deleted_at.is_some() { return Err(StreamDeletionPendingError.into()); } - let prior_doe = meta.config.delete_on_empty; + let prior_doe_min_age = meta.config.delete_on_empty.min_age(); - meta.config = resolve_stream_config( - OptionalStreamConfig::from(meta.config).reconfigure(reconfig), - basin_meta.config.default_stream_config, - ); + meta.config = OptionalStreamConfig::from(meta.config) + .reconfigure(reconfig) + .merge(basin_meta.config.default_stream_config); txn.put(&meta_key, kv::stream_meta::ser_value(&meta))?; - self.commit_stream_config(txn, basin, stream, meta.config.clone(), Some(prior_doe)) - .await?; - - Ok(meta.config) - } - - async fn commit_stream_config( - &self, - txn: DbTransaction, - basin: BasinName, - stream: StreamName, - config: StreamConfig, - prior_doe: Option, - ) -> Result<(), StorageError> { let stream_id = StreamId::new(&basin, &stream); - match (prior_doe, config.delete_on_empty.min_age()) { - (None, Some(min_age)) => { - doe::schedule(&txn, stream_id, TimestampSecs::after(min_age)).await?; - } - (Some(prior), Some(min_age)) if prior.min_age != min_age => { - doe::schedule(&txn, stream_id, TimestampSecs::now()).await?; - } - (Some(prior), None) if prior.min_age().is_some() => { - doe::clear(&txn, stream_id).await?; - } - _ => (), + if let Some(min_age) = meta.config.delete_on_empty.min_age() + && prior_doe_min_age.is_none() + { + txn.put( + kv::stream_doe_deadline::ser_key( + kv::timestamp::TimestampSecs::after(doe_arm_delay( + meta.config.retention_policy.age().unwrap_or_default(), + min_age, + )), + stream_id, + ), + kv::stream_doe_deadline::ser_value(min_age), + )?; } - let backend = self.clone(); - // Once a commit starts, cancellation must not discard its notification. - tokio::spawn(async move { - let seq = db_txn_commit_durable(txn) - .await? - .expect("stream metadata was written"); - backend.advise_stream_config(&basin, &stream, seq, config); - Ok::<_, StorageError>(()) - }) - .await - .expect("stream config commit task panicked") + txn.commit().await?; + + if let Some(client) = self.streamer_client_if_active(&basin, &stream) { + client.advise_reconfig(meta.config.clone()); + } + + Ok(meta.config) } #[instrument(ret, err, skip(self))] @@ -342,7 +350,6 @@ impl Backend { ) -> Result<(), DeleteStreamError> { self.delete_stream_with_condition(basin, stream, TerminalTrimCondition::Always) .await - .map(|_| ()) } pub(super) async fn delete_stream_with_condition( @@ -350,7 +357,7 @@ impl Backend { basin: BasinName, stream: StreamName, condition: TerminalTrimCondition, - ) -> Result { + ) -> Result<(), DeleteStreamError> { let outcome = match self.streamer_client_guarded(&basin, &stream).await { Ok(client) => client.terminal_trim(condition).await?, Err(StreamerError::Storage(e)) => { @@ -361,10 +368,10 @@ impl Backend { } Err(StreamerError::StreamDeletionPending(_)) => TerminalTrimOutcome::DeletionPending, }; - if outcome == TerminalTrimOutcome::DeletionPending { - self.mark_stream_deleted(basin, stream).await?; + match outcome { + TerminalTrimOutcome::DeletionPending => self.mark_stream_deleted(basin, stream).await, + TerminalTrimOutcome::Ineligible => Ok(()), } - Ok(outcome) } async fn mark_stream_deleted( @@ -373,49 +380,22 @@ impl Backend { stream: StreamName, ) -> Result<(), DeleteStreamError> { let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; - // A delayed deletion request may resume after the trim worker has deleted the old - // stream and the name has been reused. Only mark metadata when this - // transaction also sees a terminal trim marker. - if !has_terminal_trim(&txn, StreamId::new(&basin, &stream)).await? { - let read_seq = txn.seqnum(); - drop(txn); - // The trim worker may have removed the marker without flushing yet. - // Wait for that removal to become durable before acknowledging deletion. - self.await_durable_seq(read_seq).await?; - return Ok(()); - } let meta_key = kv::stream_meta::ser_key(&basin, &stream); - let (mut meta, seq) = db_txn_get_with(&txn, &meta_key, |entry| { - Ok((kv::stream_meta::deser_value(entry.value)?, entry.seq)) - }) - .await? - .ok_or_else(|| StreamNotFoundError { - basin, - stream: stream.clone(), - })?; + let mut meta = db_txn_get(&txn, &meta_key, kv::stream_meta::deser_value) + .await? + .ok_or_else(|| StreamNotFoundError { + basin, + stream: stream.clone(), + })?; if meta.deleted_at.is_none() { meta.deleted_at = Some(OffsetDateTime::now_utc()); txn.put(&meta_key, kv::stream_meta::ser_value(&meta))?; - db_txn_commit_durable(txn).await?; - } else { - // The terminal trim is durable, but the metadata marker may not be yet. - drop(txn); - self.await_durable_seq(seq).await?; + txn.commit().await?; } Ok(()) } } -async fn has_terminal_trim(txn: &DbTransaction, stream_id: StreamId) -> Result { - Ok(db_txn_get( - txn, - kv::stream_trim_point::ser_key(stream_id), - kv::stream_trim_point::deser_value, - ) - .await? - == Some(..NonZeroSeqNum::MAX)) -} - fn creation_idempotency_key(req_token: &RequestToken, config: &OptionalStreamConfig) -> Bash { Bash::length_prefixed(&[ req_token.as_bytes(), diff --git a/lite/src/backend/test_util.rs b/lite/src/backend/test_util.rs deleted file mode 100644 index 3c45f38a..00000000 --- a/lite/src/backend/test_util.rs +++ /dev/null @@ -1,44 +0,0 @@ -use std::future::Future; - -use slatedb::{Error, WriteHandle}; - -/// Finish a fixture write and assert that remote reads can observe it. -pub(super) trait DbWriteTestExt: - Future> + Sized -{ - async fn assert_durable(self) { - self.await - .expect("fixture write should succeed") - .await_durable() - .await - .expect("fixture write should become durable"); - } -} - -impl>> DbWriteTestExt for F {} - -/// Provision a real stream for lifecycle tests, including its ID mapping and tail. -pub(super) async fn create_stream( - backend: &super::Backend, - config: s2_common::config::OptionalStreamConfig, -) -> (s2_common::basin::BasinName, s2_common::stream::StreamName) { - use s2_common::{config::BasinConfig, resources::ProvisionMode}; - let basin: s2_common::basin::BasinName = "test-basin".parse().unwrap(); - let stream: s2_common::stream::StreamName = "test-stream".parse().unwrap(); - backend - .provision_basin(basin.clone(), BasinConfig::default(), ProvisionMode::Ensure) - .await - .unwrap(); - backend - .provision_stream( - basin.clone(), - stream.clone(), - config, - ProvisionMode::CreateOnly { - request_token: None, - }, - ) - .await - .unwrap(); - (basin, stream) -} diff --git a/lite/src/handlers/v1/error.rs b/lite/src/handlers/v1/error.rs index e589edc2..b25e34b0 100644 --- a/lite/src/handlers/v1/error.rs +++ b/lite/src/handlers/v1/error.rs @@ -177,7 +177,7 @@ impl ServiceError { } DeleteStreamError::RequestDroppedError(e) => { // Unavailable error code promised to be side-effect free, - // The terminal trim marker may have become durable prior to drop. + // AppendType::Terminal may have become durable prior to drop. standard(ErrorCode::Other, e.to_string()) } DeleteStreamError::StreamNotFound(e) => { @@ -239,7 +239,7 @@ impl ServiceError { } AppendError::RequestDroppedError(e) => { // Unavailable error code promised to be side-effect free, - // The append may have become durable prior to drop. + // AppendType::Regular may have become durable prior to drop. standard(ErrorCode::Other, e.to_string()) } AppendError::BasinNotFound(e) => standard(ErrorCode::BasinNotFound, e.to_string()), diff --git a/lite/src/handlers/v1/records.rs b/lite/src/handlers/v1/records.rs index 10fe2907..5532e20f 100644 --- a/lite/src/handlers/v1/records.rs +++ b/lite/src/handlers/v1/records.rs @@ -46,15 +46,11 @@ fn validate_read_until(start: ReadStart, end: ReadEnd) -> Result<(), ServiceErro Ok(()) } -/// Adjusts the read bounds to resume after `last_event_id`, and returns the -/// count/bytes already delivered so emitted event ids stay cumulative across -/// reconnects. fn apply_last_event_id( mut start: ReadStart, mut end: v1t::stream::ReadEnd, last_event_id: Option, -) -> (ReadStart, v1t::stream::ReadEnd, CountOrBytes) { - let mut delivered = CountOrBytes::ZERO; +) -> (ReadStart, v1t::stream::ReadEnd) { if let Some(v1t::stream::sse::LastEventId { seq_num, count, @@ -64,9 +60,8 @@ fn apply_last_event_id( start.from = ReadFrom::SeqNum(seq_num.saturating_add(1)); end.count = end.count.map(|c| c.saturating_sub(count)); end.bytes = end.bytes.map(|c| c.saturating_sub(bytes)); - delivered = CountOrBytes { count, bytes }; } - (start, end, delivered) + (start, end) } enum ReadMode { @@ -166,7 +161,6 @@ pub struct ReadArgs { v1t::StreamNamePathSegment, s2_api::data::S2FormatHeader, s2_api::data::S2EncryptionKeyHeader, - s2_api::data::S2StreamConfigHeader, v1t::stream::ReadStart, v1t::stream::ReadEnd, ), @@ -192,13 +186,12 @@ pub async fn read( match request { v1t::stream::ReadRequest::Unary { encryption_key, - create_stream_config_patch, format, response_mime, } => { let (start, end) = prepare_read(start, end, ReadMode::Unary)?; let session = backend - .open_for_read(&basin, &stream, encryption_key, create_stream_config_patch) + .open_for_read(&basin, &stream, encryption_key) .await? .read(start, end) .await?; @@ -216,19 +209,18 @@ pub async fn read( } v1t::stream::ReadRequest::EventStream { encryption_key, - create_stream_config_patch, format, last_event_id, } => { - let (start, end, delivered) = apply_last_event_id(start, end, last_event_id); + let (start, end) = apply_last_event_id(start, end, last_event_id); let (start, end) = prepare_read(start, end, ReadMode::Streaming)?; let session = backend - .open_for_read(&basin, &stream, encryption_key, create_stream_config_patch) + .open_for_read(&basin, &stream, encryption_key) .await? .read(start, end) .await?; let events = async_stream::stream! { - let mut processed = delivered; + let mut processed = CountOrBytes::ZERO; tokio::pin!(session); let mut errored = false; while let Some(output) = session.next().await { @@ -240,8 +232,8 @@ pub async fn read( let Some(last_record) = batch.records.last() else { continue; }; - processed.count = processed.count.saturating_add(batch.records.len()); - processed.bytes = processed.bytes.saturating_add(batch.records.metered_size()); + processed.count += batch.records.len(); + processed.bytes += batch.records.metered_size(); let id = v1t::stream::sse::LastEventId { seq_num: last_record.position().seq_num, count: processed.count, @@ -273,12 +265,11 @@ pub async fn read( } v1t::stream::ReadRequest::S2s { encryption_key, - create_stream_config_patch, response_compression, } => { let (start, end) = prepare_read(start, end, ReadMode::Streaming)?; let s2s_stream = backend - .open_for_read(&basin, &stream, encryption_key, create_stream_config_patch) + .open_for_read(&basin, &stream, encryption_key) .await? .read(start, end) .await? @@ -373,7 +364,6 @@ pub struct AppendArgs { v1t::StreamNamePathSegment, s2_api::data::S2FormatHeader, s2_api::data::S2EncryptionKeyHeader, - s2_api::data::S2StreamConfigHeader, ), servers( (url = super::paths::cloud_endpoints::BASIN, variables( @@ -394,12 +384,11 @@ pub async fn append( match request { v1t::stream::AppendRequest::Unary { encryption_key, - create_stream_config_patch, input, response_mime, } => { let handle = backend - .open_for_append(&basin, &stream, encryption_key, create_stream_config_patch) + .open_for_append(&basin, &stream, encryption_key) .await?; let ack = handle.append(input).await?; match response_mime { @@ -415,12 +404,11 @@ pub async fn append( } v1t::stream::AppendRequest::S2s { encryption_key, - create_stream_config_patch, inputs, response_compression, } => { let handle = backend - .open_for_append(&basin, &stream, encryption_key, create_stream_config_patch) + .open_for_append(&basin, &stream, encryption_key) .await?; let (err_tx, err_rx) = tokio::sync::oneshot::channel(); @@ -481,22 +469,16 @@ mod tests { use bytesize::ByteSize; use futures::TryStreamExt as _; use prost::Message as _; - use s2_api::v1::{ - config::STREAM_CONFIG_HEADER, - stream::{ - proto, - s2s::{self, FrameDecoder, SessionMessage}, - sse::LastEventId, - }, + use s2_api::v1::stream::{ + proto, + s2s::{FrameDecoder, SessionMessage}, }; use s2_common::{ basin::{BASIN_HEADER, BasinName}, - config::{ - BasinConfig, DeleteOnEmptyConfig, OptionalStreamConfig, RetentionPolicy, StreamConfig, - }, + config::{BasinConfig, OptionalStreamConfig}, encryption::{EncryptionAlgorithm, EncryptionKey, S2_ENCRYPTION_KEY_HEADER}, read_extent::{ReadLimit, ReadUntil}, - record::{EnvelopeRecord, Metered, MeteredSize as _, Record}, + record::{EnvelopeRecord, Metered, Record}, resources::ProvisionMode, stream::{ AppendInput, AppendRecord, AppendRecordBatch, AppendRecordParts, ListStreamsRequest, @@ -615,12 +597,7 @@ mod tests { encryption_key: EncryptionKey, ) { backend - .open_for_append( - basin, - stream, - Some(encryption_key), - OptionalStreamConfig::default(), - ) + .open_for_append(basin, stream, Some(encryption_key)) .await .expect("open append handle") .append(append_input(body)) @@ -628,95 +605,6 @@ mod tests { .expect("append encrypted payload"); } - async fn append_payload( - backend: &Backend, - basin: &BasinName, - stream: &StreamName, - body: &'static [u8], - ) { - backend - .open_for_append(basin, stream, None, OptionalStreamConfig::default()) - .await - .expect("open append handle") - .append(append_input(body)) - .await - .expect("append payload"); - } - - struct SseBatches { - seq_nums: Vec, - last_id: Option, - } - - async fn sse_read( - app: &axum::Router, - basin: &BasinName, - stream: &StreamName, - bounds: &str, - last_event_id: Option, - ) -> SseBatches { - let mut request = request_builder( - "GET", - format!("/v1/streams/{stream}/records?seq_num=0&wait=0&{bounds}"), - basin, - ) - .header(header::ACCEPT, "text/event-stream"); - if let Some(id) = last_event_id { - request = request.header("last-event-id", id.to_string()); - } - let response = send(app, request.body(Body::empty()).unwrap()).await; - assert_eq!(response.status(), StatusCode::OK); - let body = - tokio::time::timeout(Duration::from_secs(5), response_bytes(response, "sse body")) - .await - .expect("SSE read should terminate"); - let body = std::str::from_utf8(&body).expect("utf8 sse body"); - - let mut batches = SseBatches { - seq_nums: Vec::new(), - last_id: None, - }; - let mut done = false; - for event in body.split("\n\n").filter(|e| !e.trim().is_empty()) { - let field = |name: &str| { - event - .lines() - .find_map(|line| line.strip_prefix(name)) - .map(str::trim) - }; - match field("event:") { - Some("batch") => { - let id: LastEventId = field("id:") - .expect("batch event id") - .parse() - .expect("parse batch event id"); - let data: serde_json::Value = - serde_json::from_str(field("data:").expect("batch event data")) - .expect("batch event json"); - let records = data["records"].as_array().expect("records array"); - batches.seq_nums.extend( - records - .iter() - .map(|r| r["seq_num"].as_u64().expect("seq_num")), - ); - assert_eq!( - id.seq_num, - *batches.seq_nums.last().expect("non-empty batch") - ); - batches.last_id = Some(id); - } - Some("error") => panic!("unexpected sse error event: {event}"), - Some(other) => panic!("unexpected sse event `{other}`: {event}"), - None => { - assert_eq!(field("data:"), Some("[DONE]"), "unexpected event: {event}"); - done = true; - } - } - } - assert!(done, "SSE read should end with [DONE]"); - batches - } - fn read_uri(stream: &StreamName) -> String { format!("/v1/streams/{stream}/records?seq_num=0&wait=0") } @@ -809,12 +697,7 @@ mod tests { assert_eq!(ack.end.as_ref().map(|pos| pos.seq_num), Some(1)); let records = backend - .open_for_read( - &basin, - &stream, - Some(encryption_key.clone()), - OptionalStreamConfig::default(), - ) + .open_for_read(&basin, &stream, Some(encryption_key.clone())) .await .expect("open read handle") .read( @@ -848,240 +731,6 @@ mod tests { assert_eq!(record.body().as_ref(), b"secret"); } - fn basin_config_with_create_stream_on_append() -> BasinConfig { - BasinConfig { - create_stream_on_append: true, - default_stream_config: OptionalStreamConfig { - storage_class: Some("standard".into()), - ..Default::default() - }, - ..Default::default() - } - } - - fn expected_auto_created_config() -> StreamConfig { - StreamConfig { - storage_class: Some("standard".into()), - retention_policy: RetentionPolicy::Age(Duration::from_secs(3600)), - timestamping: Default::default(), - delete_on_empty: DeleteOnEmptyConfig { - min_age: Duration::from_secs(300), - }, - } - } - - const STREAM_CONFIG_HEADER_VALUE: &str = - r#"{"retention_policy":{"age":3600},"delete_on_empty":{"min_age_secs":300}}"#; - - fn append_record_input(body: &'static [u8]) -> proto::AppendInput { - proto::AppendInput { - records: vec![proto::AppendRecord { - timestamp: None, - headers: vec![], - body: Bytes::from_static(body), - }], - match_seq_num: None, - fencing_token: None, - } - } - - #[tokio::test] - async fn json_append_auto_creates_stream_with_stream_config_header() { - let (app, backend, basin, stream) = setup_app_without_stream( - "append-json-create-config", - basin_config_with_create_stream_on_append(), - ) - .await; - - let body = serde_json::json!({"records": [{"body": "hello"}]}); - let response = send( - &app, - request_builder("POST", format!("/v1/streams/{stream}/records"), &basin) - .header(header::CONTENT_TYPE, "application/json") - .header(STREAM_CONFIG_HEADER.as_str(), STREAM_CONFIG_HEADER_VALUE) - .body(Body::from(body.to_string())) - .unwrap(), - ) - .await; - - assert_eq!(response.status(), StatusCode::OK); - let ack = response_json(response, "append ack body").await; - assert_eq!(ack["end"]["seq_num"], 1); - let config = backend - .get_stream_config(basin, stream) - .await - .expect("get stream config"); - assert_eq!(config, expected_auto_created_config()); - } - - #[tokio::test] - async fn append_with_invalid_stream_config_header_is_rejected_without_creating() { - let (app, backend, basin, stream) = setup_app_without_stream( - "append-create-config-invalid", - basin_config_with_create_stream_on_append(), - ) - .await; - - for (value, expected_message) in [ - ( - r#"{"retention_policy":{"age":0}}"#, - "age must be greater than 0 seconds", - ), - ("not json", "invalid JSON"), - ] { - let body = serde_json::json!({"records": [{"body": "hello"}]}); - let response = send( - &app, - request_builder("POST", format!("/v1/streams/{stream}/records"), &basin) - .header(header::CONTENT_TYPE, "application/json") - .header(STREAM_CONFIG_HEADER.as_str(), value) - .body(Body::from(body.to_string())) - .unwrap(), - ) - .await; - - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - let info = response_json(response, "append error body").await; - assert_eq!(info["code"], "bad_header"); - let message = info["message"].as_str().expect("error message string"); - assert!( - message.contains("s2-stream-config") && message.contains(expected_message), - "{message}" - ); - } - assert_no_streams(&backend, &basin).await; - } - - #[tokio::test] - async fn s2s_append_session_auto_creates_stream_with_stream_config_header() { - let (app, backend, basin, stream) = setup_app_without_stream( - "append-s2s-create-config", - basin_config_with_create_stream_on_append(), - ) - .await; - - let frame = |body: &'static [u8]| { - SessionMessage::regular(s2s::CompressionAlgorithm::None, &append_record_input(body)) - .expect("encode frame") - .encode() - }; - let mut body = BytesMut::new(); - body.extend_from_slice(&frame(b"first")); - body.extend_from_slice(&frame(b"second")); - - let response = send( - &app, - request_builder("POST", format!("/v1/streams/{stream}/records"), &basin) - .header(header::CONTENT_TYPE, "s2s/proto") - .header(STREAM_CONFIG_HEADER.as_str(), STREAM_CONFIG_HEADER_VALUE) - .body(Body::from(body.freeze())) - .unwrap(), - ) - .await; - - assert_eq!(response.status(), StatusCode::OK); - let body = response_bytes(response, "s2s body").await; - let mut decoder = FrameDecoder; - let mut buf = BytesMut::from(body.as_ref()); - let mut acks = Vec::new(); - while let Some(frame) = decoder.decode(&mut buf).expect("frame decode") { - let SessionMessage::Regular(ack) = frame else { - panic!("expected regular frame"); - }; - acks.push( - ack.try_into_proto::() - .expect("decode append ack"), - ); - } - assert_eq!(acks.len(), 2); - assert_eq!(acks[1].end.as_ref().map(|pos| pos.seq_num), Some(2)); - - let config = backend - .get_stream_config(basin, stream) - .await - .expect("get stream config"); - assert_eq!(config, expected_auto_created_config()); - } - - #[rstest::rstest] - #[case::count("count=0")] - #[case::bytes("bytes=0")] - #[tokio::test] - async fn unary_read_with_zero_limit_returns_empty( - #[case] bound: &str, - #[values( - "timestamp=0", - "seq_num=0", - "tail_offset=0", - "timestamp=1", - "seq_num=1" - )] - start: &str, - #[values(0, 60)] wait: u32, - ) { - let (app, _backend, basin, stream) = setup_app_with_config( - "read-zero-limit", - BasinConfig::default(), - OptionalStreamConfig::default(), - ) - .await; - - let response = tokio::time::timeout( - Duration::from_secs(2), - send( - &app, - request_builder( - "GET", - format!("/v1/streams/{stream}/records?{start}&{bound}&wait={wait}"), - &basin, - ) - .body(Body::empty()) - .unwrap(), - ), - ) - .await - .expect("a zero limit should complete without waiting for records"); - - let status = response.status(); - let body = response_json(response, "read zero-limit response").await; - assert_eq!( - status, - StatusCode::OK, - "{start}&{bound}&wait={wait}: {body}" - ); - assert_eq!(body["records"], serde_json::json!([])); - } - - #[tokio::test] - async fn read_auto_creates_stream_with_stream_config_header() { - let basin_config = BasinConfig { - create_stream_on_append: false, - create_stream_on_read: true, - ..basin_config_with_create_stream_on_append() - }; - let (app, backend, basin, stream) = - setup_app_without_stream("read-create-config", basin_config).await; - - let response = send( - &app, - request_builder( - "GET", - format!("/v1/streams/{stream}/records?seq_num=0"), - &basin, - ) - .header(STREAM_CONFIG_HEADER.as_str(), STREAM_CONFIG_HEADER_VALUE) - .body(Body::empty()) - .unwrap(), - ) - .await; - assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); - let config = backend - .get_stream_config(basin, stream) - .await - .expect("get stream config"); - assert_eq!(config, expected_auto_created_config()); - } - #[tokio::test] async fn invalid_read_bounds_do_not_auto_create_stream() { let basin_config = BasinConfig { @@ -1185,103 +834,6 @@ mod tests { ); } - #[rstest::rstest] - #[case::count(Some(10), None)] - #[case::bytes(None, Some(10))] - #[case::count_first(Some(10), Some(15))] - #[case::bytes_first(Some(15), Some(10))] - #[tokio::test] - async fn sse_resume_emits_cumulative_ids_and_honors_bounds( - #[case] count: Option, - #[case] bytes_in_records: Option, - ) { - let (app, backend, basin, stream) = setup_app_with_config( - "read-sse-resume", - BasinConfig::default(), - OptionalStreamConfig::default(), - ) - .await; - let per_record = append_input(b"payload").records.metered_size(); - let mut bounds = String::new(); - if let Some(count) = count { - bounds.push_str(&format!("count={count}&")); - } - if let Some(records) = bytes_in_records { - bounds.push_str(&format!("bytes={}&", records * per_record)); - } - - // End two connections at the current tail, then resume with exactly - // the same bounds and the actual id emitted by the previous connection. - let mut last_id = None; - let mut delivered = Vec::new(); - for (appended, expected_total) in [(4, 4), (3, 7), (13, 10)] { - for _ in 0..appended { - append_payload(&backend, &basin, &stream, b"payload").await; - } - let resumed = sse_read(&app, &basin, &stream, &bounds, last_id).await; - delivered.extend(resumed.seq_nums); - assert_eq!(delivered, (0..expected_total as u64).collect::>()); - let id = resumed.last_id.expect("last id"); - assert_eq!(id.seq_num, expected_total as u64 - 1); - assert_eq!(id.count, expected_total); - assert_eq!(id.bytes, expected_total * per_record); - last_id = Some(id); - } - - // The same read without reconnects produces the same records and id. - let full = sse_read(&app, &basin, &stream, &bounds, None).await; - assert_eq!(full.seq_nums, delivered); - assert_eq!( - full.last_id.expect("full id").to_string(), - last_id.expect("resumed id").to_string(), - ); - - let exhausted = sse_read(&app, &basin, &stream, &bounds, last_id).await; - assert!(exhausted.seq_nums.is_empty(), "{:?}", exhausted.seq_nums); - assert!(exhausted.last_id.is_none()); - } - - #[rstest::rstest] - #[case::unbounded("", (usize::MAX - 1, usize::MAX - 1), (usize::MAX, usize::MAX), 3)] - #[case::count_bound("count=2", (0, usize::MAX), (2, usize::MAX), 2)] - #[case::bytes_bound("bytes=30", (usize::MAX, 0), (usize::MAX, 30), 2)] - #[tokio::test] - async fn sse_resume_saturates_client_supplied_counters( - #[case] bounds: &str, - #[case] (count, bytes): (usize, usize), - #[case] expected_totals: (usize, usize), - #[case] expected_records: usize, - ) { - let (app, backend, basin, stream) = setup_app_with_config( - "read-sse-saturation", - BasinConfig::default(), - OptionalStreamConfig::default(), - ) - .await; - // Each "payload" record occupies 15 metered bytes. - for _ in 0..4 { - append_payload(&backend, &basin, &stream, b"payload").await; - } - let resumed = sse_read( - &app, - &basin, - &stream, - bounds, - Some(LastEventId { - seq_num: 0, - count, - bytes, - }), - ) - .await; - assert_eq!( - resumed.seq_nums, - (1..=expected_records as u64).collect::>() - ); - let id = resumed.last_id.expect("last id"); - assert_eq!((id.count, id.bytes), expected_totals); - } - #[tokio::test] async fn s2s_read_without_key_header_is_rejected_before_stream_starts() { let encryption_key = aegis_key(0x42); diff --git a/lite/src/server.rs b/lite/src/server.rs index d78f71f7..bb4ec354 100644 --- a/lite/src/server.rs +++ b/lite/src/server.rs @@ -42,47 +42,15 @@ pub struct LiteArgs { /// Name of the S3 bucket to back the database. /// /// If not specified, in-memory storage is used unless --local-root is set. - /// Uses the standard AWS configuration for the endpoint, region and credentials. - #[arg(long, group = "main_store")] + #[arg(long)] pub bucket: Option, /// Root directory to back the database on the local filesystem. /// /// Conflicts with --bucket. - #[arg( - long, - value_name = "DIR", - conflicts_with = "bucket", - group = "main_store" - )] + #[arg(long, value_name = "DIR", conflicts_with = "bucket")] pub local_root: Option, - /// Name of the S3 bucket to back the write-ahead log (WAL). - /// - /// If not specified, the main store is used unless --wal-local-root is set. - /// Uses the same AWS configuration as --bucket, with optional - /// S2LITE_WAL_AWS_* overrides for the endpoint, region and credentials. - /// - /// Requires --bucket or --local-root. Conflicts with --wal-local-root. - #[arg( - long, - env = "S2LITE_WAL_BUCKET", - requires = "main_store", - conflicts_with = "wal_local_root" - )] - pub wal_bucket: Option, - - /// Root directory to back the write-ahead log (WAL) on the local filesystem. - /// - /// Requires --bucket or --local-root. Conflicts with --wal-bucket. - #[arg( - long, - env = "S2LITE_WAL_LOCAL_ROOT", - value_name = "DIR", - requires = "main_store" - )] - pub wal_local_root: Option, - /// Base path on object storage. #[arg(long, default_value = "")] pub path: String, @@ -205,44 +173,22 @@ pub async fn run(args: LiteArgs) -> eyre::Result<()> { StoreType::InMemory }; - let object_store = init_object_store(&store_type, None).await?; - let wal_store_type = if let Some(bucket) = args.wal_bucket { - Some(StoreType::S3Bucket(bucket)) - } else { - args.wal_local_root.map(StoreType::LocalFileSystem) - }; - let wal_object_store = match &wal_store_type { - Some(wal_store_type) => { - let s3_overrides = match wal_store_type { - StoreType::S3Bucket(_) => Some(WalS3Overrides::from_env()?), - StoreType::LocalFileSystem(_) | StoreType::InMemory => None, - }; - Some(init_object_store(wal_store_type, s3_overrides).await?) - } - None => None, - }; + let object_store = init_object_store(&store_type).await?; let db_settings = slatedb::Settings::from_env_with_default( "SL8_", slatedb::Settings { - flush_interval: Some( - wal_store_type - .as_ref() - .unwrap_or(&store_type) - .default_flush_interval(), - ), + flush_interval: Some(store_type.default_flush_interval()), ..Default::default() }, )?; let manifest_poll_interval = db_settings.manifest_poll_interval; - let mut builder = slatedb::Db::builder(args.path, object_store).with_settings(db_settings); - if let Some(wal_object_store) = wal_object_store { - info!(store = ?wal_store_type, "using dedicated WAL object store"); - builder = builder.with_wal_object_store(wal_object_store); - } - let db = builder.build().await?; + let db = slatedb::Db::builder(args.path, object_store) + .with_settings(db_settings) + .build() + .await?; info!( ?manifest_poll_interval, @@ -349,16 +295,58 @@ pub async fn run(args: LiteArgs) -> eyre::Result<()> { async fn init_object_store( store_type: &StoreType, - s3_overrides: Option, ) -> eyre::Result> { - let store: Arc = match store_type { + Ok(match store_type { StoreType::S3Bucket(bucket) => { info!(bucket, "using s3 object store"); - let mut builder = s3_builder().await; - if let Some(overrides) = s3_overrides { - builder = overrides.apply(builder); + let mut builder = + object_store::aws::AmazonS3Builder::from_env().with_bucket_name(bucket); + + if let Some(endpoint) = + std::env::var_os("AWS_ENDPOINT_URL_S3").and_then(|s| s.into_string().ok()) + { + if endpoint.starts_with("http://") { + builder = builder.with_allow_http(true); + } + builder = builder.with_endpoint(endpoint); } - Arc::new(builder.with_bucket_name(bucket).build()?) + + match ( + std::env::var_os("AWS_ACCESS_KEY_ID").and_then(|s| s.into_string().ok()), + std::env::var_os("AWS_SECRET_ACCESS_KEY").and_then(|s| s.into_string().ok()), + ) { + (Some(key_id), Some(secret_key)) => { + info!(key_id, "using static credentials from env vars"); + + let token = + std::env::var_os("AWS_SESSION_TOKEN").and_then(|s| s.into_string().ok()); + builder = builder.with_credentials(Arc::new( + object_store::StaticCredentialProvider::new( + object_store::aws::AwsCredential { + key_id, + secret_key, + token, + }, + ), + )); + } + _ => { + let aws_config = + aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; + if let Some(region) = aws_config.region() { + info!(region = region.as_ref()); + builder = builder.with_region(region.to_string()); + } + if let Some(credentials_provider) = aws_config.credentials_provider() { + info!("using aws-config credentials provider"); + builder = builder.with_credentials(Arc::new(S3CredentialProvider { + aws: credentials_provider.clone(), + cache: tokio::sync::Mutex::new(None), + })); + } + } + } + Arc::new(builder.build()?) as Arc } StoreType::LocalFileSystem(local_root) => { std::fs::create_dir_all(local_root)?; @@ -376,127 +364,7 @@ async fn init_object_store( info!("using in-memory object store"); Arc::new(object_store::memory::InMemory::new()) } - }; - Ok(store) -} - -// Both buckets start with the same AWS configuration and credential chain. -async fn s3_builder() -> object_store::aws::AmazonS3Builder { - let mut builder = object_store::aws::AmazonS3Builder::from_env(); - - if let Some(endpoint) = - std::env::var_os("AWS_ENDPOINT_URL_S3").and_then(|s| s.into_string().ok()) - { - if endpoint.starts_with("http://") { - builder = builder.with_allow_http(true); - } - builder = builder.with_endpoint(endpoint); - } - - match ( - std::env::var_os("AWS_ACCESS_KEY_ID").and_then(|s| s.into_string().ok()), - std::env::var_os("AWS_SECRET_ACCESS_KEY").and_then(|s| s.into_string().ok()), - ) { - (Some(key_id), Some(secret_key)) => { - info!(key_id, "using static credentials from env vars"); - - let aws_config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; - if let Some(region) = aws_config.region() { - info!(region = region.as_ref()); - builder = builder.with_region(region.to_string()); - } - - let token = std::env::var_os("AWS_SESSION_TOKEN").and_then(|s| s.into_string().ok()); - builder = builder.with_credentials(Arc::new( - object_store::StaticCredentialProvider::new(object_store::aws::AwsCredential { - key_id, - secret_key, - token, - }), - )); - } - _ => { - let aws_config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; - if let Some(region) = aws_config.region() { - info!(region = region.as_ref()); - builder = builder.with_region(region.to_string()); - } - if let Some(credentials_provider) = aws_config.credentials_provider() { - info!("using aws-config credentials provider"); - builder = builder.with_credentials(Arc::new(S3CredentialProvider { - aws: credentials_provider.clone(), - cache: tokio::sync::Mutex::new(None), - })); - } - } - } - builder -} - -// Keep credentials out of LiteArgs and its startup Debug log. Only supplied -// fields override the shared AWS configuration; credentials are replaced as a set. -struct WalS3Overrides { - endpoint: Option, - region: Option, - access_key_id: Option, - secret_access_key: Option, - session_token: Option, -} - -impl WalS3Overrides { - fn from_env() -> eyre::Result { - Self::from_getter(|name| match std::env::var(name) { - Ok(value) => Ok(Some(value)), - Err(std::env::VarError::NotPresent) => Ok(None), - Err(std::env::VarError::NotUnicode(_)) => { - Err(eyre::eyre!("{name} must contain valid UTF-8")) - } - }) - } - - fn from_getter(get: impl Fn(&str) -> eyre::Result>) -> eyre::Result { - let config = Self { - endpoint: get("S2LITE_WAL_AWS_ENDPOINT_URL_S3")?, - region: get("S2LITE_WAL_AWS_REGION")?, - access_key_id: get("S2LITE_WAL_AWS_ACCESS_KEY_ID")?, - secret_access_key: get("S2LITE_WAL_AWS_SECRET_ACCESS_KEY")?, - session_token: get("S2LITE_WAL_AWS_SESSION_TOKEN")?, - }; - eyre::ensure!( - config.access_key_id.is_some() == config.secret_access_key.is_some(), - "S2LITE_WAL_AWS_ACCESS_KEY_ID and S2LITE_WAL_AWS_SECRET_ACCESS_KEY must be set together" - ); - eyre::ensure!( - config.session_token.is_none() || config.access_key_id.is_some(), - "S2LITE_WAL_AWS_SESSION_TOKEN requires the WAL access key and secret key" - ); - Ok(config) - } - - fn apply( - self, - mut builder: object_store::aws::AmazonS3Builder, - ) -> object_store::aws::AmazonS3Builder { - if let Some(endpoint) = &self.endpoint { - // Override the S3-specific endpoint inherited from the main store. - builder = builder - .with_allow_http(endpoint.starts_with("http://")) - .with_config(object_store::aws::AmazonS3ConfigKey::S3Endpoint, endpoint); - } - if let Some(region) = self.region { - builder = builder.with_region(region); - } - if let (Some(key_id), Some(secret_key)) = (self.access_key_id, self.secret_access_key) { - builder = builder.with_credentials(Arc::new( - object_store::StaticCredentialProvider::new(object_store::aws::AwsCredential { - key_id, - secret_key, - token: self.session_token, - }), - )); - } - builder - } + }) } async fn shutdown_signal(handle: axum_server::Handle) { @@ -591,33 +459,7 @@ impl object_store::CredentialProvider for S3CredentialProvider { #[cfg(test)] mod tests { - use super::{ServerProtocol, WalS3Overrides, cli_endpoint, cli_env_hint}; - - fn wal_config(values: &[(&str, &str)]) -> eyre::Result { - WalS3Overrides::from_getter(|name| { - Ok(values - .iter() - .find(|(key, _)| *key == name) - .map(|(_, value)| (*value).to_owned())) - }) - } - - #[test] - fn wal_static_credentials_must_be_complete() { - for values in [ - vec![("S2LITE_WAL_AWS_ACCESS_KEY_ID", "test-key")], - vec![("S2LITE_WAL_AWS_SECRET_ACCESS_KEY", "do-not-log-this-secret")], - vec![("S2LITE_WAL_AWS_SESSION_TOKEN", "do-not-log-this-token")], - ] { - let error = wal_config(&values) - .err() - .expect("invalid credentials") - .to_string(); - assert!(error.contains("S2LITE_WAL_AWS_")); - assert!(!error.contains("do-not-log-this")); - } - assert!(wal_config(&[]).is_ok()); - } + use super::{ServerProtocol, cli_endpoint, cli_env_hint}; #[test] fn cli_endpoint_uses_localhost_with_explicit_port() { diff --git a/lite/tests/backend/common/mod.rs b/lite/tests/backend/common/mod.rs index a4bea287..019728e9 100644 --- a/lite/tests/backend/common/mod.rs +++ b/lite/tests/backend/common/mod.rs @@ -3,7 +3,6 @@ use std::pin::Pin; use futures::Stream; use s2_common::{ basin::BasinName, - config::OptionalStreamConfig, encryption::EncryptionSpec, record::StreamPosition, stream::{AppendAck, AppendInput, StreamName}, @@ -31,7 +30,6 @@ pub async fn append( &basin, &stream, encryption.and_then(encryption_key_for_spec), - OptionalStreamConfig::default(), ) .await? .append(input) @@ -53,7 +51,6 @@ where &basin, &stream, encryption.and_then(encryption_key_for_spec), - OptionalStreamConfig::default(), ) .await? .append_session(inputs); diff --git a/lite/tests/backend/common/read.rs b/lite/tests/backend/common/read.rs index 6e3405ba..28cb6b33 100644 --- a/lite/tests/backend/common/read.rs +++ b/lite/tests/backend/common/read.rs @@ -3,7 +3,6 @@ use std::{pin::Pin, task::Poll, time::Duration}; use futures::StreamExt; use s2_common::{ basin::BasinName, - config::OptionalStreamConfig, encryption::EncryptionSpec, read_extent::{ReadLimit, ReadUntil}, record::{Record, SequencedRecord}, @@ -84,12 +83,7 @@ pub async fn try_open_read_session_with_encryption( ReadError, > { let read_session = backend - .open_for_read( - basin, - stream, - encryption_key_for_spec(encryption), - OptionalStreamConfig::default(), - ) + .open_for_read(basin, stream, encryption_key_for_spec(encryption)) .await? .read(start, end) .await?; diff --git a/lite/tests/backend/common/setup.rs b/lite/tests/backend/common/setup.rs index 597f8023..324ea3b6 100644 --- a/lite/tests/backend/common/setup.rs +++ b/lite/tests/backend/common/setup.rs @@ -1,4 +1,4 @@ -use std::{fmt::Debug, future::Future, sync::Arc, time::Duration}; +use std::{sync::Arc, time::Duration}; use bytes::Bytes; use bytesize::ByteSize; @@ -20,16 +20,12 @@ const TEST_AEGIS256_KEY: [u8; 32] = [0x42; 32]; const TEST_AES256_GCM_KEY: [u8; 32] = [0x24; 32]; pub async fn create_in_memory_db() -> Db { - create_in_memory_db_with_flush_interval(Some(Duration::from_millis(5))).await -} - -async fn create_in_memory_db_with_flush_interval(flush_interval: Option) -> Db { let object_store = Arc::new(InMemory::new()); let db_path = format!("/tmp/test_{}", Uuid::new_v4()); Db::builder(db_path, object_store) .with_settings(Settings { - flush_interval, + flush_interval: Some(Duration::from_millis(5)), ..Default::default() }) .build() @@ -42,50 +38,6 @@ pub async fn create_backend() -> Backend { Backend::new(db, ByteSize::mib(10)) } -pub async fn create_backend_without_auto_flush() -> (Backend, Db) { - let db = create_in_memory_db_with_flush_interval(None).await; - (Backend::new(db.clone(), ByteSize::mib(10)), db) -} - -/// Poll an operation until the database has a committed, unflushed write. -/// The database must have automatic flushing disabled and no unrelated writers. -pub async fn assert_pending_until_committed( - db: &Db, - operation: &mut (impl Future + Unpin), -) -> u64 { - tokio::time::timeout(Duration::from_secs(5), async { - tokio::select! { - biased; - result = operation => panic!("metadata acknowledged before flush: {result:?}"), - seq = async { - loop { - let seq = db.snapshot().await.unwrap().seq(); - if seq > db.status().durable_seq { - return seq; - } - tokio::task::yield_now().await; - } - } => seq, - } - }) - .await - .expect("metadata should be committed in memory before flushing") -} - -pub async fn assert_waits_for_flush(db: &Db, operation: impl Future) -> T { - tokio::pin!(operation); - tokio::time::timeout(Duration::from_secs(5), async { - let seq = assert_pending_until_committed(db, &mut operation).await; - assert!(futures::poll!(&mut operation).is_pending()); - db.flush().await.unwrap(); - let result = operation.await; - assert!(db.status().durable_seq >= seq); - result - }) - .await - .expect("metadata write should finish after an explicit flush") -} - pub fn test_basin_name(suffix: &str) -> BasinName { format!("test-basin-{}", suffix).parse().unwrap() } @@ -285,12 +237,7 @@ pub async fn append_payloads_with_encryption( fencing_token: None, }; backend - .open_for_append( - basin, - stream, - encryption_key_for_spec(encryption), - OptionalStreamConfig::default(), - ) + .open_for_append(basin, stream, encryption_key_for_spec(encryption)) .await .expect("Failed to open append handle") .append(input) @@ -310,7 +257,7 @@ pub async fn append_timestamped_payloads( fencing_token: None, }; backend - .open_for_append(basin, stream, None, OptionalStreamConfig::default()) + .open_for_append(basin, stream, None) .await .expect("Failed to open append handle") .append(input) diff --git a/lite/tests/backend/control_plane/basin.rs b/lite/tests/backend/control_plane/basin.rs index 67e1037d..f5d09f22 100644 --- a/lite/tests/backend/control_plane/basin.rs +++ b/lite/tests/backend/control_plane/basin.rs @@ -4,7 +4,8 @@ use s2_common::{ basin::{BasinNamePrefix, BasinNameStartAfter, ListBasinsRequest}, config::{ BasinConfig, BasinReconfiguration, OptionalDeleteOnEmptyConfig, OptionalStreamConfig, - RetentionPolicy, StreamReconfiguration, TimestampingMode, TimestampingReconfiguration, + RetentionPolicy, StorageClass, StreamReconfiguration, TimestampingMode, + TimestampingReconfiguration, }, maybe::Maybe, resources::{ProvisionMode, ProvisionResult, RequestToken}, @@ -15,80 +16,6 @@ use s2_lite::backend::error::{ use super::common::*; -#[tokio::test] -async fn test_provision_basin_acknowledges_only_durable_metadata() { - let (backend, db) = create_backend_without_auto_flush().await; - let basin = test_basin_name("durable"); - let result = assert_waits_for_flush( - &db, - backend.provision_basin(basin.clone(), BasinConfig::default(), ProvisionMode::Ensure), - ) - .await - .unwrap(); - assert!(matches!(result, ProvisionResult::Created(_))); - assert_eq!( - backend.get_basin_config(basin).await.unwrap(), - BasinConfig::default() - ); - backend.close().await.unwrap(); -} - -#[rstest::rstest] -#[case::ensure(ProvisionMode::Ensure, false)] -#[case::idempotent_create(ProvisionMode::CreateOnly { - request_token: Some("original".parse().unwrap()), -}, false)] -#[case::create_without_token(ProvisionMode::CreateOnly { - request_token: None, -}, true)] -#[case::create_with_different_token(ProvisionMode::CreateOnly { - request_token: Some("different".parse().unwrap()), -}, true)] -#[tokio::test] -async fn test_basin_retries_acknowledge_only_durable_metadata( - #[case] retry_mode: ProvisionMode, - #[case] expect_already_exists: bool, -) { - let (backend, db) = create_backend_without_auto_flush().await; - let basin = test_basin_name("durable-retry"); - let creation = backend.provision_basin( - basin.clone(), - BasinConfig::default(), - ProvisionMode::CreateOnly { - request_token: Some("original".parse().unwrap()), - }, - ); - tokio::pin!(creation); - assert_pending_until_committed(&db, &mut creation).await; - assert!(matches!( - backend.get_basin_config(basin.clone()).await, - Err(GetBasinConfigError::BasinNotFound(_)) - )); - - let retry = assert_waits_for_flush( - &db, - backend.provision_basin(basin.clone(), BasinConfig::default(), retry_mode), - ) - .await; - if expect_already_exists { - assert!(matches!( - retry, - Err(ProvisionBasinError::BasinAlreadyExists(_)) - )); - } else { - assert!(matches!(retry, Ok(ProvisionResult::Noop(_)))); - } - assert!(matches!( - creation.await.unwrap(), - ProvisionResult::Created(_) - )); - assert_eq!( - backend.get_basin_config(basin).await.unwrap(), - BasinConfig::default() - ); - backend.close().await.unwrap(); -} - #[tokio::test] async fn test_create_basin_idempotency_respects_request_token() { let backend = create_backend().await; @@ -240,7 +167,7 @@ async fn test_provision_basin_ensure_updates_config() { let mut updated_config = initial_config.clone(); updated_config.create_stream_on_append = true; updated_config.create_stream_on_read = true; - updated_config.default_stream_config.storage_class = Some("standard".into()); + updated_config.default_stream_config.storage_class = Some(StorageClass::Standard); backend .provision_basin( @@ -259,7 +186,7 @@ async fn test_provision_basin_ensure_updates_config() { assert!(stored_config.create_stream_on_read); assert_eq!( stored_config.default_stream_config.storage_class, - Some("standard".into()) + Some(StorageClass::Standard) ); backend @@ -282,7 +209,7 @@ async fn test_provision_basin_ensure_resets_unspecified_config() { create_stream_on_append: true, create_stream_on_read: false, default_stream_config: OptionalStreamConfig { - storage_class: Some("standard".into()), + storage_class: Some(StorageClass::Standard), retention_policy: Some(RetentionPolicy::Infinite()), ..Default::default() }, @@ -369,7 +296,7 @@ async fn test_reconfigure_basin_updates_nested_defaults() { let backend = create_backend().await; let basin_name = test_basin_name("basin-reconfigure"); let mut initial_config = BasinConfig::default(); - initial_config.default_stream_config.storage_class = Some("standard".into()); + initial_config.default_stream_config.storage_class = Some(StorageClass::Standard); backend .provision_basin( @@ -387,7 +314,7 @@ async fn test_reconfigure_basin_updates_nested_defaults() { ..Default::default() }; let mut stream_reconfig = StreamReconfiguration { - storage_class: Maybe::from(Some("express".into())), + storage_class: Maybe::from(Some(StorageClass::Express)), retention_policy: Maybe::from(Some(RetentionPolicy::Infinite())), ..Default::default() }; @@ -409,7 +336,7 @@ async fn test_reconfigure_basin_updates_nested_defaults() { assert!(updated.create_stream_on_read); assert_eq!( updated.default_stream_config.storage_class, - Some("express".into()) + Some(StorageClass::Express) ); assert_eq!( updated.default_stream_config.retention_policy, @@ -426,7 +353,7 @@ async fn test_reconfigure_basin_updates_nested_defaults() { .expect("Failed to fetch basin config after reconfigure"); assert_eq!( fetched.default_stream_config.storage_class, - Some("express".into()) + Some(StorageClass::Express) ); assert_eq!( fetched.default_stream_config.retention_policy, @@ -440,30 +367,6 @@ async fn test_reconfigure_basin_updates_nested_defaults() { assert!(fetched.create_stream_on_read); } -#[tokio::test(start_paused = true)] -async fn test_delete_basin_retry_waits_for_durable_metadata() { - let (backend, db) = create_backend_without_auto_flush().await; - let basin = assert_waits_for_flush( - &db, - create_test_basin(&backend, "durable-delete", BasinConfig::default()), - ) - .await; - - let mut first = Box::pin(backend.delete_basin(basin.clone())); - assert_pending_until_committed(&db, &mut first).await; - drop(first); - - let retry = backend.delete_basin(basin); - tokio::pin!(retry); - assert!( - tokio::time::timeout(Duration::from_secs(1), &mut retry) - .await - .is_err() - ); - assert_waits_for_flush(&db, retry).await.unwrap(); - backend.close().await.unwrap(); -} - #[tokio::test] async fn test_delete_basin_marks_deleting_and_blocks_create() { let backend = create_backend().await; diff --git a/lite/tests/backend/control_plane/stream.rs b/lite/tests/backend/control_plane/stream.rs index 6ded3547..a3ca62c5 100644 --- a/lite/tests/backend/control_plane/stream.rs +++ b/lite/tests/backend/control_plane/stream.rs @@ -5,7 +5,8 @@ use s2_common::{ config::{ BasinConfig, BasinReconfiguration, DeleteOnEmptyReconfiguration, OptionalDeleteOnEmptyConfig, OptionalStreamConfig, OptionalTimestampingConfig, - RetentionPolicy, StreamReconfiguration, TimestampingMode, TimestampingReconfiguration, + RetentionPolicy, StorageClass, StreamReconfiguration, TimestampingMode, + TimestampingReconfiguration, }, encryption::EncryptionAlgorithm, maybe::Maybe, @@ -22,34 +23,6 @@ use s2_lite::backend::error::{ use super::common::*; -#[tokio::test] -async fn test_provision_stream_acknowledges_only_durable_metadata() { - let (backend, db) = create_backend_without_auto_flush().await; - let basin = test_basin_name("durable-stream"); - let stream = test_stream_name("durable"); - assert_waits_for_flush( - &db, - backend.provision_basin(basin.clone(), BasinConfig::default(), ProvisionMode::Ensure), - ) - .await - .unwrap(); - - let result = assert_waits_for_flush( - &db, - backend.provision_stream( - basin.clone(), - stream.clone(), - OptionalStreamConfig::default(), - ProvisionMode::Ensure, - ), - ) - .await - .unwrap(); - assert!(matches!(result, ProvisionResult::Created(_))); - backend.get_stream_config(basin, stream).await.unwrap(); - backend.close().await.unwrap(); -} - #[tokio::test] async fn test_create_stream_honors_basin_defaults() { let backend = create_backend().await; @@ -57,7 +30,7 @@ async fn test_create_stream_honors_basin_defaults() { let basin_config = BasinConfig { default_stream_config: OptionalStreamConfig { - storage_class: Some("standard".into()), + storage_class: Some(StorageClass::Standard), retention_policy: Some(RetentionPolicy::Infinite()), timestamping: OptionalTimestampingConfig { mode: Some(TimestampingMode::ClientRequire), @@ -97,7 +70,7 @@ async fn test_create_stream_honors_basin_defaults() { .get_stream_config(basin_name, stream_name) .await .expect("Failed to fetch stream config"); - assert_eq!(config.storage_class.as_deref(), Some("standard")); + assert_eq!(config.storage_class, StorageClass::Standard); assert_eq!(config.retention_policy, RetentionPolicy::Infinite()); assert_eq!(config.timestamping.mode, TimestampingMode::ClientRequire); } @@ -239,7 +212,7 @@ async fn test_create_stream_idempotency_and_request_token() { let stream_name = test_stream_name("stream-idempotency"); let config = OptionalStreamConfig { - storage_class: Some("express".into()), + storage_class: Some(StorageClass::Express), ..Default::default() }; @@ -261,7 +234,7 @@ async fn test_create_stream_idempotency_and_request_token() { .get_stream_config(basin_name.clone(), stream_name.clone()) .await .expect("Failed to fetch stored stream config"); - assert_eq!(stored_config.storage_class.as_deref(), Some("express")); + assert_eq!(stored_config.storage_class, StorageClass::Express); let idempotent = backend .provision_stream( @@ -325,7 +298,7 @@ async fn test_provision_stream_ensure_preserves_idempotency_key() { let stream_name = test_stream_name("stream-idempotency-key-preserve"); let config = OptionalStreamConfig { - storage_class: Some("standard".into()), + storage_class: Some(StorageClass::Standard), ..Default::default() }; let token: RequestToken = "stream-token-preserve".parse().unwrap(); @@ -374,7 +347,7 @@ async fn test_provision_stream_ensure_preserves_idempotency_key() { .get_stream_config(basin_name.clone(), stream_name.clone()) .await .expect("Failed to fetch stream config"); - assert_eq!(stored_config.storage_class.as_deref(), Some("express")); + assert_eq!(stored_config.storage_class, StorageClass::Express); assert_eq!(stored_config.timestamping.mode, TimestampingMode::Arrival); backend @@ -398,7 +371,7 @@ async fn test_provision_stream_ensure_noops_when_effective_config_matches() { "stream-ensure-effective-noop", BasinConfig { default_stream_config: OptionalStreamConfig { - storage_class: Some("express".into()), + storage_class: Some(StorageClass::Express), retention_policy: Some(RetentionPolicy::Age(Duration::from_secs( 10 * 24 * 60 * 60, ))), @@ -410,7 +383,7 @@ async fn test_provision_stream_ensure_noops_when_effective_config_matches() { .await; let stream_name = test_stream_name("stream-ensure-effective-noop"); let config = OptionalStreamConfig { - storage_class: Some("standard".into()), + storage_class: Some(StorageClass::Standard), retention_policy: Some(RetentionPolicy::Infinite()), ..Default::default() }; @@ -516,7 +489,7 @@ async fn test_provision_stream_idempotency_ignores_changed_basin_defaults() { basin_name.clone(), BasinReconfiguration { default_stream_config: Maybe::from(Some(StreamReconfiguration { - storage_class: Maybe::from(Some("standard".into())), + storage_class: Maybe::from(Some(StorageClass::Standard)), ..Default::default() })), ..Default::default() @@ -544,7 +517,7 @@ async fn test_reconfigure_stream_updates_selected_fields() { let basin_name = test_basin_name("stream-reconfigure"); let mut basin_config = BasinConfig::default(); - basin_config.default_stream_config.storage_class = Some("standard".into()); + basin_config.default_stream_config.storage_class = Some(StorageClass::Standard); backend .provision_basin( @@ -584,7 +557,7 @@ async fn test_reconfigure_stream_updates_selected_fields() { uncapped: Maybe::from(Some(true)), }; let mut stream_reconfig = StreamReconfiguration { - storage_class: Maybe::from(Some("express".into())), + storage_class: Maybe::from(Some(StorageClass::Express)), retention_policy: Maybe::from(Some(RetentionPolicy::Infinite())), ..Default::default() }; @@ -595,7 +568,7 @@ async fn test_reconfigure_stream_updates_selected_fields() { .await .expect("Failed to reconfigure stream"); - assert_eq!(updated.storage_class.as_deref(), Some("express")); + assert_eq!(updated.storage_class, StorageClass::Express); assert_eq!(updated.retention_policy, RetentionPolicy::Infinite()); assert_eq!(updated.timestamping.mode, TimestampingMode::Arrival); assert!(updated.timestamping.uncapped); @@ -604,7 +577,7 @@ async fn test_reconfigure_stream_updates_selected_fields() { .get_stream_config(basin_name, stream_name) .await .expect("Failed to fetch stream config after reconfigure"); - assert_eq!(fetched.storage_class.as_deref(), Some("express")); + assert_eq!(fetched.storage_class, StorageClass::Express); assert_eq!(fetched.retention_policy, RetentionPolicy::Infinite()); assert_eq!(fetched.timestamping.mode, TimestampingMode::Arrival); assert!(fetched.timestamping.uncapped); @@ -617,7 +590,7 @@ async fn test_reconfigure_stream_clears_fields_to_basin_defaults() { let basin_config = BasinConfig { default_stream_config: OptionalStreamConfig { - storage_class: Some("standard".into()), + storage_class: Some(StorageClass::Standard), retention_policy: Some(RetentionPolicy::Infinite()), timestamping: OptionalTimestampingConfig { mode: Some(TimestampingMode::Arrival), @@ -643,7 +616,7 @@ async fn test_reconfigure_stream_clears_fields_to_basin_defaults() { let stream_name = test_stream_name("stream-reconfigure-clear-defaults"); let stream_config = OptionalStreamConfig { - storage_class: Some("express".into()), + storage_class: Some(StorageClass::Express), retention_policy: Some(RetentionPolicy::Age(Duration::from_secs(60))), timestamping: OptionalTimestampingConfig { mode: Some(TimestampingMode::ClientRequire), @@ -683,7 +656,7 @@ async fn test_reconfigure_stream_clears_fields_to_basin_defaults() { .await .expect("Failed to reconfigure stream"); - assert_eq!(updated.storage_class.as_deref(), Some("standard")); + assert_eq!(updated.storage_class, StorageClass::Standard); assert_eq!(updated.retention_policy, RetentionPolicy::Infinite()); assert_eq!(updated.timestamping.mode, TimestampingMode::Arrival); assert!(updated.timestamping.uncapped); @@ -697,101 +670,84 @@ async fn test_reconfigure_stream_clears_fields_to_basin_defaults() { assert_eq!(fetched, updated); } -#[rstest::rstest] -#[case::patch(false, false)] -#[case::ensure(true, false)] -#[case::cancelled_patch(false, true)] -#[case::cancelled_ensure(true, true)] #[tokio::test] -async fn test_stream_config_updates_active_streamer(#[case] ensure: bool, #[case] cancel: bool) { - let (backend, db) = create_backend_without_auto_flush().await; - let basin = assert_waits_for_flush( - &db, - create_test_basin(&backend, "config-delivery", BasinConfig::default()), +async fn test_reconfigure_stream_updates_active_streamer() { + let (backend, basin_name, stream_name) = setup_backend_with_stream( + "stream-reconfigure-active", + "stream", + OptionalStreamConfig::default(), ) .await; - let stream = assert_waits_for_flush( - &db, - create_test_stream( - &backend, - &basin, - "stream", - OptionalStreamConfig { - timestamping: OptionalTimestampingConfig { - mode: Some(TimestampingMode::ClientRequire), - ..Default::default() - }, - ..Default::default() - }, - ), - ) - .await; - let _active_streamer = backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) - .await - .unwrap(); - let mut update = Box::pin(async { - if ensure { - backend - .provision_stream( - basin.clone(), - stream.clone(), - OptionalStreamConfig { - timestamping: OptionalTimestampingConfig { - mode: Some(TimestampingMode::Arrival), - ..Default::default() - }, - ..Default::default() - }, - ProvisionMode::Ensure, - ) - .await - .unwrap(); - } else { - backend - .reconfigure_stream( - basin.clone(), - stream.clone(), - StreamReconfiguration { - timestamping: Maybe::from(Some(TimestampingReconfiguration { - mode: Maybe::from(Some(TimestampingMode::Arrival)), - ..Default::default() - })), - ..Default::default() - }, - ) - .await - .unwrap(); - } - }); - assert_pending_until_committed(&db, &mut update).await; - if cancel { - drop(update); - db.flush().await.unwrap(); - } else { - db.flush().await.unwrap(); - update.await; - } + append_payloads(&backend, &basin_name, &stream_name, &[b"seed"]).await; - // Application is asynchronous; retry until the active streamer accepts arrival timestamps. - let append_after_update = async { - loop { - let input = AppendInput { - records: create_test_record_batch(vec![Bytes::from_static(b"no timestamp")]), - match_seq_num: None, - fencing_token: None, - }; - match append(&backend, basin.clone(), stream.clone(), input, None).await { - Err(AppendError::TimestampMissing(_)) => tokio::task::yield_now().await, - result => break result, - } - } + let ts_reconfig = TimestampingReconfiguration { + mode: Maybe::from(Some(TimestampingMode::ClientRequire)), + uncapped: Maybe::default(), }; - assert_waits_for_flush(&db, append_after_update) + let reconfig = StreamReconfiguration { + timestamping: Maybe::from(Some(ts_reconfig)), + ..Default::default() + }; + + backend + .reconfigure_stream(basin_name.clone(), stream_name.clone(), reconfig) .await - .unwrap(); - backend.close().await.unwrap(); + .expect("Failed to reconfigure stream"); + + check_tail(&backend, basin_name.clone(), stream_name.clone()) + .await + .expect("Failed to check tail"); + + let input = AppendInput { + records: create_test_record_batch(vec![Bytes::from_static(b"missing timestamp")]), + match_seq_num: None, + fencing_token: None, + }; + let result = append(&backend, basin_name, stream_name, input, None).await; + assert!(matches!(result, Err(AppendError::TimestampMissing(_)))); +} + +#[tokio::test] +async fn test_provision_stream_ensure_updates_active_streamer() { + let (backend, basin_name, stream_name) = setup_backend_with_stream( + "stream-ensure-active", + "stream", + OptionalStreamConfig::default(), + ) + .await; + + append_payloads(&backend, &basin_name, &stream_name, &[b"seed"]).await; + + let config = OptionalStreamConfig { + timestamping: OptionalTimestampingConfig { + mode: Some(TimestampingMode::ClientRequire), + ..Default::default() + }, + ..Default::default() + }; + + backend + .provision_stream( + basin_name.clone(), + stream_name.clone(), + config, + ProvisionMode::Ensure, + ) + .await + .expect("Ensure should succeed for an existing stream"); + + check_tail(&backend, basin_name.clone(), stream_name.clone()) + .await + .expect("Failed to check tail"); + + let input = AppendInput { + records: create_test_record_batch(vec![Bytes::from_static(b"missing timestamp")]), + match_seq_num: None, + fencing_token: None, + }; + let result = append(&backend, basin_name, stream_name, input, None).await; + assert!(matches!(result, Err(AppendError::TimestampMissing(_)))); } #[tokio::test] @@ -823,38 +779,6 @@ async fn test_create_stream_fails_when_basin_deleting() { )); } -#[tokio::test(start_paused = true)] -async fn test_delete_stream_retry_waits_for_durable_metadata() { - let (backend, db) = create_backend_without_auto_flush().await; - let basin = assert_waits_for_flush( - &db, - create_test_basin(&backend, "durable-delete", BasinConfig::default()), - ) - .await; - let stream = assert_waits_for_flush( - &db, - create_test_stream(&backend, &basin, "durable-delete", Default::default()), - ) - .await; - - let mut first = Box::pin(backend.delete_stream(basin.clone(), stream.clone())); - // Stream deletion first persists the terminal trim, then the metadata marker. - assert_pending_until_committed(&db, &mut first).await; - db.flush().await.unwrap(); - assert_pending_until_committed(&db, &mut first).await; - drop(first); - - let retry = backend.delete_stream(basin, stream); - tokio::pin!(retry); - assert!( - tokio::time::timeout(Duration::from_secs(1), &mut retry) - .await - .is_err() - ); - assert_waits_for_flush(&db, retry).await.unwrap(); - backend.close().await.unwrap(); -} - #[tokio::test] async fn test_delete_stream_marks_deleted_and_blocks_recreation() { let backend = create_backend().await; diff --git a/lite/tests/backend/data_plane/auto_create.rs b/lite/tests/backend/data_plane/auto_create.rs index 491a3eb7..24bdfc00 100644 --- a/lite/tests/backend/data_plane/auto_create.rs +++ b/lite/tests/backend/data_plane/auto_create.rs @@ -3,10 +3,7 @@ use std::time::Duration; use bytes::Bytes; use s2_common::{ basin::BasinName, - config::{ - BasinConfig, DeleteOnEmptyConfig, OptionalDeleteOnEmptyConfig, OptionalStreamConfig, - RetentionPolicy, StreamConfig, - }, + config::BasinConfig, encryption::EncryptionAlgorithm, read_extent::{ReadLimit, ReadUntil}, record::StreamPosition, @@ -126,140 +123,6 @@ async fn test_backend_append_auto_creates_stream_with_basin_cipher() { assert_eq!(envelope_bodies(&records), vec![b"secret".to_vec()]); } -fn basin_config_with_defaults() -> BasinConfig { - BasinConfig { - create_stream_on_append: true, - default_stream_config: OptionalStreamConfig { - storage_class: Some("standard".into()), - retention_policy: Some(RetentionPolicy::Age(Duration::from_secs(7 * 24 * 60 * 60))), - ..Default::default() - }, - ..Default::default() - } -} - -fn requested_stream_config() -> OptionalStreamConfig { - OptionalStreamConfig { - retention_policy: Some(RetentionPolicy::Age(Duration::from_secs(3600))), - delete_on_empty: OptionalDeleteOnEmptyConfig { - min_age: Some(Duration::from_secs(300)), - }, - ..Default::default() - } -} - -fn expected_merged_stream_config() -> StreamConfig { - StreamConfig { - storage_class: Some("standard".into()), - retention_policy: RetentionPolicy::Age(Duration::from_secs(3600)), - timestamping: Default::default(), - delete_on_empty: DeleteOnEmptyConfig { - min_age: Duration::from_secs(300), - }, - } -} - -#[tokio::test] -async fn test_backend_append_auto_create_applies_stream_config() { - let backend = create_backend().await; - let basin_name = create_test_basin( - &backend, - "backend-auto-create-config", - basin_config_with_defaults(), - ) - .await; - let stream_name = test_stream_name("missing"); - - let input = AppendInput { - records: create_test_record_batch(vec![Bytes::from_static(b"hello")]), - match_seq_num: None, - fencing_token: None, - }; - let ack = backend - .open_for_append(&basin_name, &stream_name, None, requested_stream_config()) - .await - .expect("Failed to open append handle") - .append(input) - .await - .expect("Failed to append to auto-created stream"); - assert_eq!(ack.end.seq_num, 1); - - let config = backend - .get_stream_config(basin_name.clone(), stream_name.clone()) - .await - .expect("Failed to get stream config"); - assert_eq!(config, expected_merged_stream_config()); -} - -#[tokio::test] -async fn test_backend_append_ignores_stream_config_for_existing_stream() { - let backend = create_backend().await; - let basin_name = create_test_basin( - &backend, - "backend-auto-create-config-existing", - basin_config_with_defaults(), - ) - .await; - let stream_name = create_test_stream( - &backend, - &basin_name, - "existing", - OptionalStreamConfig::default(), - ) - .await; - let before = backend - .get_stream_config(basin_name.clone(), stream_name.clone()) - .await - .expect("Failed to get stream config"); - - let input = AppendInput { - records: create_test_record_batch(vec![Bytes::from_static(b"hello")]), - match_seq_num: None, - fencing_token: None, - }; - backend - .open_for_append(&basin_name, &stream_name, None, requested_stream_config()) - .await - .expect("Failed to open append handle") - .append(input) - .await - .expect("Failed to append to existing stream"); - - let after = backend - .get_stream_config(basin_name.clone(), stream_name.clone()) - .await - .expect("Failed to get stream config"); - assert_eq!(after, before); - assert_ne!(after, expected_merged_stream_config()); -} - -#[tokio::test] -async fn test_backend_read_auto_create_applies_stream_config() { - let backend = create_backend().await; - let basin_name = create_test_basin( - &backend, - "backend-auto-create-read-config", - BasinConfig { - create_stream_on_append: false, - create_stream_on_read: true, - ..basin_config_with_defaults() - }, - ) - .await; - let stream_name = test_stream_name("missing"); - - backend - .open_for_read(&basin_name, &stream_name, None, requested_stream_config()) - .await - .expect("Failed to open read handle on auto-created stream"); - - let config = backend - .get_stream_config(basin_name.clone(), stream_name.clone()) - .await - .expect("Failed to get stream config"); - assert_eq!(config, expected_merged_stream_config()); -} - #[tokio::test] async fn test_backend_append_without_auto_create_returns_not_found() { let backend = create_backend().await; diff --git a/lite/tests/backend/data_plane/mixed.rs b/lite/tests/backend/data_plane/mixed.rs index 7bc8c40c..63b9a372 100644 --- a/lite/tests/backend/data_plane/mixed.rs +++ b/lite/tests/backend/data_plane/mixed.rs @@ -2,7 +2,7 @@ use std::{sync::Arc, time::Duration}; use bytes::Bytes; use s2_common::{ - config::{OptionalStreamConfig, RetentionPolicy, StreamReconfiguration}, + config::{OptionalStreamConfig, RetentionPolicy, StorageClass, StreamReconfiguration}, read_extent::{ReadLimit, ReadUntil}, stream::{AppendInput, ReadEnd, ReadFrom, ReadStart}, }; @@ -147,7 +147,7 @@ async fn test_concurrent_reconfigure_during_append() { ready.notified().await; let reconfig = StreamReconfiguration { - storage_class: s2_common::maybe::Maybe::from(Some("express".into())), + storage_class: s2_common::maybe::Maybe::from(Some(StorageClass::Express)), retention_policy: s2_common::maybe::Maybe::from(Some(RetentionPolicy::Infinite())), timestamping: s2_common::maybe::Maybe::default(), delete_on_empty: s2_common::maybe::Maybe::default(), @@ -157,7 +157,7 @@ async fn test_concurrent_reconfigure_during_append() { .reconfigure_stream(basin_name.clone(), stream_name.clone(), reconfig) .await .expect("Failed to reconfigure stream during appends"); - assert_eq!(updated_config.storage_class.as_deref(), Some("express")); + assert_eq!(updated_config.storage_class, StorageClass::Express); append_handle.await.unwrap(); diff --git a/lite/tests/backend/data_plane/read.rs b/lite/tests/backend/data_plane/read.rs index 511ae50c..900d89ee 100644 --- a/lite/tests/backend/data_plane/read.rs +++ b/lite/tests/backend/data_plane/read.rs @@ -326,31 +326,6 @@ async fn test_read_at_tail_without_follow_returns_unwritten( } } -#[rstest] -#[case::seq_num(ReadFrom::SeqNum(0))] -#[case::timestamp(ReadFrom::Timestamp(0))] -#[tokio::test] -async fn test_read_at_tail_of_empty_stream_returns_unwritten(#[case] from: ReadFrom) { - let (backend, basin_name, stream_name) = setup_backend_with_stream( - "read-empty-at-tail", - "stream", - OptionalStreamConfig::default(), - ) - .await; - - let start = ReadStart { from, clamp: false }; - let end = tail_read_end(TailEndCase::CountNoWait); - let result = try_open_read_session(&backend, &basin_name, &stream_name, start, end).await; - - match result { - Err(ReadError::Unwritten(UnwrittenError(tail))) => { - assert_eq!(tail, StreamPosition::MIN); - } - Ok(_) => panic!("Expected Unwritten error for {from:?} on an empty stream, got Ok"), - Err(e) => panic!("Expected Unwritten error for {from:?} on an empty stream, got: {e:?}"), - } -} - #[tokio::test] async fn test_read_from_tail_offset() { let (backend, basin_name, stream_name) = setup_backend_with_stream( diff --git a/resource-spec/CHANGELOG.md b/resource-spec/CHANGELOG.md index d5368b6d..745d023e 100644 --- a/resource-spec/CHANGELOG.md +++ b/resource-spec/CHANGELOG.md @@ -2,14 +2,6 @@ All notable changes to this project will be documented in this file. -## [0.3.0] - 2026-09-25 - -### Features - -- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) - - - ## [0.2.2] - 2026-07-07 ### Miscellaneous Tasks diff --git a/resource-spec/Cargo.toml b/resource-spec/Cargo.toml index 4c5816c3..1c0ca47d 100644 --- a/resource-spec/Cargo.toml +++ b/resource-spec/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-resource-spec" -version = "0.3.0" +version = "0.2.2" description = "Declarative resource specifications for S2" edition.workspace = true license.workspace = true @@ -9,7 +9,6 @@ homepage.workspace = true keywords = ["s2", "durable", "streams", "config", "schema"] [dependencies] -compact_str = { workspace = true, features = ["serde"] } humantime = { workspace = true } s2-common = { workspace = true } schemars = { workspace = true } diff --git a/resource-spec/src/lib.rs b/resource-spec/src/lib.rs index 042edd6c..7a0ea7fa 100644 --- a/resource-spec/src/lib.rs +++ b/resource-spec/src/lib.rs @@ -2,7 +2,6 @@ use std::{borrow::Cow, time::Duration}; -use compact_str::CompactString; use s2_common::{basin::BasinName, stream::StreamName}; use serde::{Deserialize, Serialize}; @@ -51,10 +50,9 @@ pub struct BasinConfig { #[derive(Debug, Clone, Deserialize, Default, schemars::JsonSchema)] #[serde(deny_unknown_fields)] pub struct StreamConfig { - /// [Storage class](https://s2.dev/docs/storage-classes) for recent writes. + /// Storage class for recent writes. #[serde(default)] - #[schemars(with = "Option")] - pub storage_class: Option, + pub storage_class: Option, /// Retention policy for the stream. If unspecified, the default is to retain records for 7 /// days. #[serde(default)] @@ -67,6 +65,36 @@ pub struct StreamConfig { pub delete_on_empty: Option, } +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum StorageClass { + Standard, + Express, +} + +impl schemars::JsonSchema for StorageClass { + fn schema_name() -> Cow<'static, str> { + "StorageClass".into() + } + + fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "string", + "description": "Storage class for recent writes.", + "enum": ["standard", "express"] + }) + } +} + +impl From for s2_common::config::StorageClass { + fn from(s: StorageClass) -> Self { + match s { + StorageClass::Standard => Self::Standard, + StorageClass::Express => Self::Express, + } + } +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub enum EncryptionAlgorithm { #[serde(rename = "aegis-256")] @@ -259,7 +287,7 @@ impl From for s2_common::config::OptionalDeleteOnEmptyConfig { impl From for s2_common::config::OptionalStreamConfig { fn from(s: StreamConfig) -> Self { Self { - storage_class: s.storage_class, + storage_class: s.storage_class.map(Into::into), retention_policy: s.retention_policy.map(|rp| rp.0), timestamping: s.timestamping.map(Into::into).unwrap_or_default(), delete_on_empty: s.delete_on_empty.map(Into::into).unwrap_or_default(), @@ -323,9 +351,8 @@ pub fn validate(spec: &Resources) -> Result<(), String> { } fn validate_stream_config(config: &StreamConfig, context: &str, errors: &mut Vec) { - if let Some(retention_policy) = config.retention_policy - && let Err(err) = retention_policy.0.validate() - { + let config = s2_common::config::OptionalStreamConfig::from(config.clone()); + if let Err(err) = config.validate() { errors.push(format!("{context}: {err}")); } } @@ -432,7 +459,7 @@ mod tests { assert_eq!(config.create_stream_on_read, Some(false)); let dsc = config.default_stream_config.as_ref().unwrap(); - assert_eq!(dsc.storage_class.as_deref(), Some("express")); + assert!(matches!(dsc.storage_class, Some(StorageClass::Express))); assert!(matches!( dsc.retention_policy.as_ref().map(|r| &r.0), Some(s2_common::config::RetentionPolicy::Age(_)) @@ -452,7 +479,7 @@ mod tests { let stream = &basin.streams[0]; assert_eq!(stream.name.as_ref(), "events"); let sc = stream.config.as_ref().unwrap(); - assert_eq!(sc.storage_class.as_deref(), Some("standard")); + assert!(matches!(sc.storage_class, Some(StorageClass::Standard))); assert!(matches!( sc.retention_policy.as_ref().map(|r| &r.0), Some(s2_common::config::RetentionPolicy::Infinite()) @@ -577,7 +604,7 @@ mod tests { #[test] fn stream_config_conversion() { let spec = StreamConfig { - storage_class: Some("standard".into()), + storage_class: Some(StorageClass::Standard), retention_policy: Some(RetentionPolicy( s2_common::config::RetentionPolicy::Infinite(), )), @@ -585,7 +612,10 @@ mod tests { delete_on_empty: None, }; let config = s2_common::config::OptionalStreamConfig::from(spec); - assert_eq!(config.storage_class, Some("standard".into())); + assert_eq!( + config.storage_class, + Some(s2_common::config::StorageClass::Standard) + ); assert_eq!( config.retention_policy, Some(s2_common::config::RetentionPolicy::Infinite()) diff --git a/sdk/CHANGELOG.md b/sdk/CHANGELOG.md index ed7c6a3b..8d07c476 100644 --- a/sdk/CHANGELOG.md +++ b/sdk/CHANGELOG.md @@ -2,58 +2,6 @@ All notable changes to this project will be documented in this file. -## [0.35.0] - 2026-09-25 - -### Features - -- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) - - - -## [0.34.10] - 2026-09-24 - -### Bug Fixes - -- Preserve uncertainty across append retries ([#767](https://github.com/s2-streamstore/s2/issues/767)) - - - -## [0.34.9] - 2026-09-22 - - - -## [0.34.8] - 2026-09-16 - -### Bug Fixes - -- Reject Content-Type in default headers ([#740](https://github.com/s2-streamstore/s2/issues/740)) - - - -## [0.34.7] - 2026-09-11 - -### Features - -- `s2-stream-config` header for auto-created streams ([#718](https://github.com/s2-streamstore/s2/issues/718)) - - - -## [0.34.6] - 2026-09-11 - -### Features - -- Set default request headers (`_hidden` only) ([#731](https://github.com/s2-streamstore/s2/issues/731)) - - - -## [0.34.5] - 2026-09-10 - -### Bug Fixes - -- Reject NUL bytes in stream names and access token IDs ([#728](https://github.com/s2-streamstore/s2/issues/728)) - - - ## [0.34.4] - 2026-09-01 ### Features diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml index db739f37..ecdd764f 100644 --- a/sdk/Cargo.toml +++ b/sdk/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "s2-sdk" description = "Rust SDK for S2" -version = "0.35.0" +version = "0.34.4" edition.workspace = true license.workspace = true repository = "https://github.com/s2-streamstore/s2/tree/main/sdk" @@ -25,7 +25,6 @@ async-compression = { version = "0.4", features = ["tokio", "gzip", "zstd"] } async-stream = { workspace = true } async-trait = { workspace = true } bytes = { workspace = true } -compact_str = { workspace = true, features = ["serde"] } futures-core = { workspace = true } futures-util = { workspace = true } h2 = "0.4" @@ -47,7 +46,7 @@ serde_json = { workspace = true } serde_urlencoded = "0.7" thiserror = { workspace = true } time = { workspace = true } -tokio = { version = "1.53", features = ["time", "macros", "io-util", "sync"] } +tokio = { version = "1.6", features = ["time", "macros", "io-util", "sync"] } tokio-muxt = "0.7" tokio-stream = { workspace = true } tokio-util = { workspace = true, features = ["rt", "codec"] } diff --git a/sdk/examples/reconfigure_basin.rs b/sdk/examples/reconfigure_basin.rs index a1946d28..45a1488d 100644 --- a/sdk/examples/reconfigure_basin.rs +++ b/sdk/examples/reconfigure_basin.rs @@ -1,8 +1,8 @@ use s2_sdk::{ S2, types::{ - BasinName, BasinReconfiguration, ReconfigureBasinInput, S2Config, StreamReconfiguration, - TimestampingReconfiguration, + BasinName, BasinReconfiguration, ReconfigureBasinInput, S2Config, StorageClass, + StreamReconfiguration, TimestampingReconfiguration, }, }; @@ -22,7 +22,7 @@ async fn main() -> Result<(), Box> { BasinReconfiguration::new() .with_default_stream_config( StreamReconfiguration::new() - .with_storage_class("standard") + .with_storage_class(StorageClass::Standard) .with_timestamping(TimestampingReconfiguration::new().with_uncapped(true)), ) .with_create_stream_on_read(true), diff --git a/sdk/src/api.rs b/sdk/src/api.rs index 84175479..e518babc 100644 --- a/sdk/src/api.rs +++ b/sdk/src/api.rs @@ -18,10 +18,7 @@ use s2_api::v1::{ basin::{ BasinInfo, CreateBasinRequest, EnsureBasinRequest, ListBasinsRequest, ListBasinsResponse, }, - config::{ - BasinConfig, BasinReconfiguration, STREAM_CONFIG_HEADER, StreamConfig, - StreamReconfiguration, - }, + config::{BasinConfig, BasinReconfiguration, StreamConfig, StreamReconfiguration}, location::LocationInfo, metrics::{ AccountMetricSetRequest, BasinMetricSetRequest, MetricSetResponse, StreamMetricSetRequest, @@ -46,7 +43,7 @@ use crate::{ error::{ClientError, server_error_has_no_side_effects, server_error_is_retryable}, frame_signal::FrameSignal, reconnect::ReconnectAdvice, - retry::{AppendRetryError, RetryBackoff, RetryBackoffBuilder}, + retry::{RetryBackoff, RetryBackoffBuilder}, types::{ AccessToken, AccessTokenId, AccessTokenMode, AppendRetryPolicy, BasinAuthority, BasinName, Compression, EncryptionKey, LocationName, RetryConfig, S2Config, S2Endpoints, StreamName, @@ -387,7 +384,6 @@ impl BasinClient { name: &StreamName, input: AppendInput, encryption: Option<&EncryptionKey>, - stream_config: Option<&StreamConfig>, append_retry_policy: AppendRetryPolicy, ) -> Result { let url = self.uri(format!("v1/streams/{}/records", urlencoding::encode(name))); @@ -398,7 +394,6 @@ impl BasinClient { .body(input.encode_to_vec()) .build()?; set_encryption_header(&mut request, encryption); - set_stream_config_header(&mut request, stream_config); let response = self .request(request) .with_append_retry_policy(append_retry_policy) @@ -425,7 +420,6 @@ impl BasinClient { start: ReadStart, end: ReadEnd, encryption: Option<&EncryptionKey>, - stream_config: Option<&StreamConfig>, ) -> Result { let url = self.uri(format!("v1/streams/{}/records", urlencoding::encode(name))); let mut builder = self @@ -439,7 +433,6 @@ impl BasinClient { } let mut request = builder.build()?; set_encryption_header(&mut request, encryption); - set_stream_config_header(&mut request, stream_config); let response = self .request(request) .error_handler(read_response_error_handler) @@ -453,7 +446,6 @@ impl BasinClient { name: &StreamName, inputs: I, encryption: Option<&EncryptionKey>, - stream_config: Option<&StreamConfig>, frame_signal: Option, reconnect: ReconnectAdvice, ) -> Result, ApiError> @@ -484,7 +476,6 @@ impl BasinClient { add_basin_header_if_required(request_builder, &self.config.endpoints, &self.name); let mut request = request_builder.build()?; set_encryption_header(&mut request, encryption); - set_stream_config_header(&mut request, stream_config); let (response, access_token) = self.client.init_streaming_authorized(request).await?; let response = match response.into_result().await { Ok(response) => response, @@ -547,7 +538,6 @@ impl BasinClient { start: ReadStart, end: ReadEnd, encryption: Option<&EncryptionKey>, - stream_config: Option<&StreamConfig>, reconnect: ReconnectAdvice, ) -> Result, ApiError> { let url = self.uri(format!("v1/streams/{}/records", urlencoding::encode(name))); @@ -563,7 +553,6 @@ impl BasinClient { add_basin_header_if_required(request_builder, &self.config.endpoints, &self.name); let mut request = request_builder.build()?; set_encryption_header(&mut request, encryption); - set_stream_config_header(&mut request, stream_config); let (response, access_token) = self.client.init_streaming_authorized(request).await?; let response = match response.into_result().await { Ok(response) => response, @@ -667,13 +656,6 @@ pub(crate) enum ApiError { Compression(#[from] std::io::Error), #[error("append condition check failed")] AppendConditionFailed(AppendConditionFailed), - #[error( - "append may have taken effect in an earlier attempt; final attempt failed: {final_attempt_error}" - )] - IndefiniteFailure { - #[source] - final_attempt_error: Box, - }, #[error("read from an unwritten position")] ReadUnwritten(TailResponse), #[error("{1}")] @@ -685,9 +667,6 @@ impl ApiError { match self { Self::Server(status, err_resp) => server_error_is_retryable(*status, &err_resp.code), Self::Client(err) => err.is_retryable(), - Self::IndefiniteFailure { - final_attempt_error, - } => final_attempt_error.is_retryable(), #[cfg(feature = "_hidden")] Self::AccessTokenProvider(error) => error.is_retryable(), _ => false, @@ -708,27 +687,18 @@ impl ApiError { Self::Server(StatusCode::UNAUTHORIZED, response) if response.code == "authn" ) } -} -impl AppendRetryError for ApiError { - fn has_no_side_effects(&self) -> bool { + pub fn has_no_side_effects(&self) -> bool { match self { Self::Server(status, err_resp) => { server_error_has_no_side_effects(*status, &err_resp.code) } Self::Client(err) => err.has_no_side_effects(), - Self::AppendConditionFailed(_) | Self::MalformedAccessToken(_) => true, #[cfg(feature = "_hidden")] Self::AccessTokenProvider(_) => true, _ => false, } } - - fn into_indefinite_failure(self) -> Self { - Self::IndefiniteFailure { - final_attempt_error: Box::new(self), - } - } } impl From for ApiError { @@ -824,10 +794,7 @@ impl BaseClient { C: client::Connect + Clone + Send + Sync + 'static, { let access_token_mode = config.access_token.mode(); - let mut default_headers = config.default_headers.clone(); - // Authorization belongs to the configured token, including when a - // refreshable provider supplies it immediately before each attempt. - default_headers.remove(AUTHORIZATION); + let mut default_headers = HeaderMap::new(); #[cfg(feature = "_hidden")] let mut access_token_provider = None; match &config.access_token { @@ -998,14 +965,6 @@ fn set_encryption_header(request: &mut client::Request, encryption: Option<&Encr } } -fn set_stream_config_header(request: &mut client::Request, stream_config: Option<&StreamConfig>) { - if let Some(config) = stream_config { - request - .headers_mut() - .insert(STREAM_CONFIG_HEADER.clone(), config.to_header_value()); - } -} - pub fn retry_builder(config: &RetryConfig) -> RetryBackoffBuilder { RetryBackoffBuilder::default() .with_min_base_delay(config.min_base_delay) @@ -1054,7 +1013,6 @@ impl<'a> RequestBuilder<'a> { let mut retry_backoff: Option = self .retry_enabled .then(|| self.client.retry_builder.build()); - let mut prior_uncertainty = false; loop { if let Some(ref signal) = self.frame_signal { @@ -1132,11 +1090,6 @@ impl<'a> RequestBuilder<'a> { num_retries_remaining = retry_backoff.as_ref().map(|b| b.remaining()).unwrap_or(0), "retrying request" ); - if self.append_retry_policy.is_some() - && err.attempt_may_have_side_effects(self.frame_signal.as_ref()) - { - prior_uncertainty = true; - } tokio::time::sleep(backoff).await; } else { debug!( @@ -1146,7 +1099,7 @@ impl<'a> RequestBuilder<'a> { retries_exhausted = retry_backoff.as_ref().is_none_or(|b| b.is_exhausted()), "not retrying request" ); - return Err(err.with_prior_uncertainty(prior_uncertainty)); + return Err(err); } } } @@ -1160,7 +1113,9 @@ fn is_safe_to_retry( ) -> bool { let policy_compliant = match policy { None | Some(AppendRetryPolicy::All) => true, - Some(AppendRetryPolicy::NoSideEffects) => !err.attempt_may_have_side_effects(frame_signal), + Some(AppendRetryPolicy::NoSideEffects) => { + !frame_signal.is_none_or(|s| s.is_signalled()) || err.has_no_side_effects() + } }; policy_compliant && (err.is_retryable() @@ -1306,11 +1261,11 @@ fn provision_result_from_parts( #[cfg(test)] mod tests { - use std::error::Error; #[cfg(feature = "_hidden")] - use std::sync::Mutex; - #[cfg(feature = "_hidden")] - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::{ + Mutex, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }; #[cfg(feature = "_hidden")] use async_trait::async_trait; @@ -1318,243 +1273,6 @@ mod tests { use hyper_util::client::legacy::connect::HttpConnector; use super::*; - use crate::error::AppendError; - - #[cfg(feature = "_hidden")] - #[derive(Default)] - struct HeaderCapture { - unary: Mutex>, - streaming: Mutex>, - } - - #[cfg(feature = "_hidden")] - #[async_trait] - impl client::RequestExecutor for HeaderCapture { - async fn execute_unary( - &self, - mut request: client::Request, - ) -> Result { - let mut headers = self.unary.lock().unwrap(); - headers.push(request.headers_mut().clone()); - // Exercise a real retry through RequestBuilder::send. - Ok(if headers.len() == 1 { - UnaryResponse::new_for_test( - StatusCode::SERVICE_UNAVAILABLE, - br#"{"code":"unavailable","message":"retry"}"#.to_vec(), - ) - } else { - UnaryResponse::new_for_test( - StatusCode::OK, - br#"{"basins":[],"streams":[],"has_more":false}"#.to_vec(), - ) - }) - } - - async fn init_streaming( - &self, - mut request: client::Request, - ) -> Result { - self.streaming - .lock() - .unwrap() - .push(request.headers_mut().clone()); - // Capturing initiation is sufficient: do not construct a response body. - Err(client::HttpError::Timeout) - } - } - - #[cfg(feature = "_hidden")] - #[tokio::test] - async fn default_headers_reach_account_basin_streaming_and_retry_requests() { - let mut session = HeaderValue::from_static("session-1"); - session.set_sensitive(true); - let headers = HeaderMap::from_iter([ - ( - http::header::HeaderName::from_static("x-origin-session"), - session, - ), - ( - AUTHORIZATION, - HeaderValue::from_static("Bearer wrong-token"), - ), - ( - http::header::USER_AGENT, - HeaderValue::from_static("wrong-agent"), - ), - ( - http::header::ACCEPT_ENCODING, - HeaderValue::from_static("wrong-encoding"), - ), - ( - http::header::HeaderName::from_static(S2_BASIN), - HeaderValue::from_static("wrong-basin"), - ), - ]); - let config = S2Config::new("actual-token") - .with_endpoints(S2Endpoints::for_endpoint("http://example.test").unwrap()) - .with_compression(Compression::Gzip) - .with_default_headers(headers) - .unwrap(); - let executor = Arc::new(HeaderCapture::default()); - let mut base = BaseClient::init_with_connector(&config, HttpConnector::new()).unwrap(); - base.client = executor.clone(); - base.retry_builder = RetryBackoffBuilder::default() - .with_min_base_delay(Duration::ZERO) - .with_max_base_delay(Duration::ZERO) - .with_max_retries(1); - let account = AccountClient::init(config.clone(), base); - account - .list_basins(ListBasinsRequest { - prefix: None, - start_after: None, - limit: None, - }) - .await - .unwrap(); - let basin = account.basin_client("test-basin".parse().unwrap()); - basin - .list_streams(ListStreamsRequest { - prefix: None, - start_after: None, - limit: None, - }) - .await - .unwrap(); - - let stream = "test-stream".parse().unwrap(); - assert!( - basin - .read_session( - &stream, - ReadStart { - seq_num: None, - timestamp: None, - tail_offset: None, - clamp: None - }, - ReadEnd { - count: None, - bytes: None, - until: None, - wait: None - }, - None, - None, - ReconnectAdvice::default(), - ) - .await - .is_err() - ); - assert!( - basin - .append_session( - &stream, - futures_util::stream::empty(), - None, - None, - None, - ReconnectAdvice::default(), - ) - .await - .is_err() - ); - - let unary = executor.unary.lock().unwrap(); - let streaming = executor.streaming.lock().unwrap(); - assert_eq!(unary.len(), 3); // account, account retry, basin - assert_eq!(streaming.len(), 2); // read and append - for headers in unary.iter().chain(streaming.iter()) { - assert_eq!(headers["x-origin-session"], "session-1"); - assert!(headers["x-origin-session"].is_sensitive()); - assert_eq!(headers[AUTHORIZATION], "Bearer actual-token"); - assert!(headers[AUTHORIZATION].is_sensitive()); - assert_eq!(headers[http::header::USER_AGENT], config.user_agent); - assert_eq!(headers[http::header::ACCEPT_ENCODING], "gzip"); - assert!(!headers.contains_key(http::header::CONTENT_ENCODING)); - } - assert_eq!(unary[2][S2_BASIN], "test-basin"); - for headers in streaming.iter() { - assert_eq!(headers[S2_BASIN], "test-basin"); - assert_eq!(headers[CONTENT_TYPE], CONTENT_TYPE_S2S); - } - } - - #[cfg(feature = "_hidden")] - #[rstest::rstest] - #[case::none(Compression::None, None, "gzip, zstd")] - #[case::gzip(Compression::Gzip, Some("gzip"), "gzip")] - #[case::zstd(Compression::Zstd, Some("zstd"), "zstd")] - #[tokio::test] - async fn default_accept_encoding_respects_configured_compression( - #[case] compression: Compression, - #[case] content_encoding: Option<&str>, - #[case] accept_encoding: &str, - ) { - let config = S2Config::new("token") - .with_compression(compression) - .with_default_headers(HeaderMap::from_iter([( - http::header::ACCEPT_ENCODING, - HeaderValue::from_static("gzip, zstd"), - )])) - .unwrap(); - let client = BaseClient::init_with_connector(&config, HttpConnector::new()).unwrap(); - let mut request = client - .post("http://example.test/v1/basins".parse().unwrap()) - .json(&serde_json::json!({"name": "test-basin"})) - .build() - .unwrap() - .compress() - .await - .unwrap(); - let headers = request.headers_mut(); - assert_eq!( - headers - .get(http::header::CONTENT_ENCODING) - .map(|value| value.to_str().unwrap()), - content_encoding - ); - assert_eq!(headers[http::header::ACCEPT_ENCODING], accept_encoding); - } - - #[cfg(feature = "_hidden")] - #[tokio::test] - async fn default_headers_are_replaced_and_isolated_between_clients() { - let first_headers = HeaderMap::from_iter([ - ( - http::header::HeaderName::from_static("x-origin-session"), - HeaderValue::from_static("first"), - ), - ( - http::header::HeaderName::from_static("x-first-only"), - HeaderValue::from_static("present"), - ), - ]); - let first = S2Config::new("token") - .with_default_headers(first_headers) - .unwrap(); - let second = first - .clone() - .with_default_headers(HeaderMap::from_iter([( - http::header::HeaderName::from_static("x-origin-session"), - HeaderValue::from_static("second"), - )])) - .unwrap(); - for (config, session) in [(&first, "first"), (&second, "second")] { - let client = BaseClient::init_with_connector(config, HttpConnector::new()).unwrap(); - let mut request = client - .get("http://example.test".parse().unwrap()) - .build() - .unwrap(); - assert_eq!(request.headers_mut()["x-origin-session"], session); - assert_eq!( - request.headers_mut().contains_key("x-first-only"), - session == "first" - ); - } - let cleared = second.with_default_headers(HeaderMap::new()).unwrap(); - assert!(cleared.default_headers.is_empty()); - assert!(S2Config::new("token").default_headers.is_empty()); - } #[cfg(feature = "_hidden")] #[derive(Debug)] @@ -1641,78 +1359,6 @@ mod tests { } } - #[rstest::rstest] - #[case(StatusCode::FORBIDDEN, "permission_denied", false, false)] - #[case(StatusCode::FORBIDDEN, "permission_denied", true, true)] - #[case(StatusCode::SERVICE_UNAVAILABLE, "unavailable", false, false)] - #[case(StatusCode::SERVICE_UNAVAILABLE, "unavailable", true, false)] - #[case(StatusCode::TOO_MANY_REQUESTS, "rate_limited", true, true)] - #[test] - fn unary_append_failure_preserves_uncertainty_and_final_error( - #[case] status: StatusCode, - #[case] code: &str, - #[case] prior_uncertainty: bool, - #[case] wrapped: bool, - ) { - let error = - AppendError::from(server_error(status, code).with_prior_uncertainty(prior_uncertainty)); - assert_eq!( - matches!(error, AppendError::IndefiniteFailure { .. }), - wrapped - ); - let server = error.request_error().unwrap().server_error().unwrap(); - assert_eq!((server.status, server.code.as_str()), (status, code)); - assert_eq!(server.message, "test"); - assert_eq!(error.is_retryable(), server.is_retryable()); - assert_eq!( - error.has_no_side_effects(), - !wrapped && server.has_no_side_effects() - ); - if wrapped { - let latest = error - .source() - .unwrap() - .downcast_ref::>() - .unwrap(); - assert!(latest.has_no_side_effects()); - assert_eq!( - latest.request_error().unwrap().server_error().unwrap().code, - code - ); - assert_eq!( - error.to_string(), - format!( - "append may have taken effect in an earlier attempt; final attempt failed: {latest}" - ) - ); - } - } - - #[rstest::rstest] - #[case::condition_failed( - ApiError::AppendConditionFailed(AppendConditionFailed::SeqNumMismatch(42)), - "sequence number mismatch, expected: 42" - )] - #[case::malformed_access_token( - ApiError::MalformedAccessToken("invalid header".to_owned()), - "malformed access token: invalid header" - )] - #[test] - fn unary_append_wraps_definite_terminal_errors(#[case] error: ApiError, #[case] message: &str) { - assert!(error.has_no_side_effects()); - let error = AppendError::from(error.with_prior_uncertainty(true)); - assert!(matches!(error, AppendError::IndefiniteFailure { .. })); - assert!(!error.has_no_side_effects()); - assert!(!error.is_retryable()); - let latest = error - .source() - .unwrap() - .downcast_ref::>() - .unwrap(); - assert!(latest.has_no_side_effects()); - assert_eq!(latest.to_string(), message); - } - fn server_error(status: StatusCode, code: &str) -> ApiError { ApiError::Server( status, @@ -1789,15 +1435,9 @@ mod tests { #[cfg(feature = "_hidden")] #[tokio::test] async fn dynamic_access_token_is_loaded_for_each_attempt_and_marked_sensitive() { - let config = S2Config::new("unused") - .with_default_headers(HeaderMap::from_iter([( - AUTHORIZATION, - HeaderValue::from_static("Bearer wrong-token"), - )])) - .unwrap() - .with_access_token_provider(RotatingTokenProvider { - generation: AtomicUsize::new(1), - }); + let config = S2Config::new("unused").with_access_token_provider(RotatingTokenProvider { + generation: AtomicUsize::new(1), + }); let client = BaseClient::init_with_connector(&config, HttpConnector::new()).unwrap(); let uri = "http://example.test/v1/basins".parse().unwrap(); let mut request = client.get(uri).build().unwrap(); diff --git a/sdk/src/batching.rs b/sdk/src/batching.rs index eac2c5ab..00086f1f 100644 --- a/sdk/src/batching.rs +++ b/sdk/src/batching.rs @@ -174,7 +174,6 @@ impl Stream for AppendInputs { records: batch, match_seq_num, fencing_token: self.fencing_token.clone(), - stream_config: None, }))) } Poll::Ready(Some(Err(err))) => Poll::Ready(Some(Err(err))), diff --git a/sdk/src/client.rs b/sdk/src/client.rs index abf927ab..b38d3f31 100644 --- a/sdk/src/client.rs +++ b/sdk/src/client.rs @@ -304,8 +304,9 @@ impl RequestBuilder { } pub fn headers(mut self, headers: &HeaderMap) -> Self { - // An owned HeaderMap replaces each key's existing values while preserving duplicates. - self.headers.extend(headers.clone()); + for (key, value) in headers { + self.headers.insert(key.clone(), value.clone()); + } self } @@ -573,7 +574,11 @@ fn build_http_request( let mut builder = http::Request::builder().method(method).uri(uri.clone()); if let Some(req_headers) = builder.headers_mut() { - *req_headers = headers; + for (key, value) in headers { + if let Some(key) = key { + req_headers.insert(key, value); + } + } if let Some(encoding) = content_encoding { req_headers.insert(CONTENT_ENCODING, encoding); } @@ -1011,37 +1016,6 @@ mod tests { Pool::new(HttpConnector::new()) } - #[test] - fn default_headers_preserve_multiple_values_and_sensitive_flags_on_wire_request() { - let mut headers = HeaderMap::new(); - headers.append("x-tag", HeaderValue::from_static("first")); - let mut second = HeaderValue::from_static("second"); - second.set_sensitive(true); - headers.append("x-tag", second); - headers.insert(CONTENT_ENCODING, HeaderValue::from_static("wrong-encoding")); - - let request = RequestBuilder::get("http://example.test".parse().unwrap()) - .header("x-tag", "replaced") - .headers(&headers) - .build() - .unwrap(); - let cloned = request.try_clone().unwrap(); - let http = build_http_request( - cloned.method, - &cloned.uri, - cloned.headers, - cloned.body.into_http_body(), - Some(HeaderValue::from_static("gzip")), - ) - .unwrap(); - let values = http.headers().get_all("x-tag").iter().collect::>(); - assert_eq!(values.len(), 2); - assert_eq!(values[0], "first"); - assert_eq!(values[1], "second"); - assert!(values[1].is_sensitive()); - assert_eq!(http.headers()[CONTENT_ENCODING], "gzip"); - } - #[test] fn uri_with_path_percent_encoded_segment() { let base: Uri = "https://example.com".parse().unwrap(); diff --git a/sdk/src/error.rs b/sdk/src/error.rs index bdeb5e53..7fd07726 100644 --- a/sdk/src/error.rs +++ b/sdk/src/error.rs @@ -332,28 +332,12 @@ pub enum AppendError { /// The append condition did not match. #[error(transparent)] ConditionFailed(#[from] AppendConditionFailed), - /// The final attempt failed definitively, but an earlier attempt may have taken effect, - /// so the entire append operation is indeterminate. - #[error( - "append may have taken effect in an earlier attempt; final attempt failed: {final_attempt_error}" - )] - IndefiniteFailure { - /// The definite error returned by the final attempt. - #[source] - final_attempt_error: Box, - }, } impl AppendError { /// Whether retrying the operation is safe or sensible. pub fn is_retryable(&self) -> bool { - match self { - Self::Request(error) => error.is_retryable(), - Self::ConditionFailed(_) => false, - Self::IndefiniteFailure { - final_attempt_error, - } => final_attempt_error.is_retryable(), - } + matches!(self, Self::Request(error) if error.is_retryable()) } /// Whether retrying the operation cannot duplicate a mutation. @@ -361,7 +345,6 @@ impl AppendError { match self { Self::Request(error) => error.has_no_side_effects(), Self::ConditionFailed(_) => true, - Self::IndefiniteFailure { .. } => false, } } @@ -370,9 +353,6 @@ impl AppendError { match self { Self::Request(error) => Some(error), Self::ConditionFailed(_) => None, - Self::IndefiniteFailure { - final_attempt_error, - } => final_attempt_error.request_error(), } } } @@ -381,11 +361,6 @@ impl From for AppendError { fn from(error: ApiError) -> Self { match error { ApiError::AppendConditionFailed(condition) => Self::ConditionFailed(condition.into()), - ApiError::IndefiniteFailure { - final_attempt_error, - } => Self::IndefiniteFailure { - final_attempt_error: Box::new((*final_attempt_error).into()), - }, other => Self::Request(other.into()), } } diff --git a/sdk/src/ops.rs b/sdk/src/ops.rs index db339709..be287bec 100644 --- a/sdk/src/ops.rs +++ b/sdk/src/ops.rs @@ -4,9 +4,7 @@ use crate::{ api::{AccountClient, BaseClient, BasinClient}, error::{AppendError, ReadError, RequestError}, producer::{Producer, ProducerConfig}, - session::{ - self, AppendSession, AppendSessionConfig, ReadSession, ReadSessionError, StreamHeaders, - }, + session::{self, AppendSession, AppendSessionConfig, ReadSession, ReadSessionError}, types::{ AccessTokenId, AccessTokenInfo, AppendAck, AppendInput, BasinConfig, BasinInfo, BasinName, CreateBasinInput, CreateStreamInput, DeleteBasinInput, DeleteStreamInput, EncryptionKey, @@ -439,13 +437,6 @@ impl S2Stream { } } - fn headers(&self, stream_config: Option<&StreamConfig>) -> StreamHeaders { - StreamHeaders { - encryption: self.encryption.clone(), - stream_config: stream_config.cloned().map(Into::into), - } - } - /// Check tail position. pub async fn check_tail(&self) -> Result { let response = self.client.check_tail(&self.name).await?; @@ -453,18 +444,13 @@ impl S2Stream { } /// Append records. - pub async fn append(&self, mut input: AppendInput) -> Result { - let stream_config = input - .stream_config - .take() - .map(s2_api::v1::config::StreamConfig::from); + pub async fn append(&self, input: AppendInput) -> Result { let ack = self .client .append( &self.name, input.into(), self.encryption.as_ref(), - stream_config.as_ref(), self.client.config.retry.append_retry_policy, ) .await?; @@ -473,9 +459,6 @@ impl S2Stream { /// Read records. pub async fn read(&self, input: ReadInput) -> Result { - let stream_config = input - .stream_config - .map(s2_api::v1::config::StreamConfig::from); let batch = self .client .read( @@ -483,7 +466,6 @@ impl S2Stream { input.start.into(), input.stop.into(), self.encryption.as_ref(), - stream_config.as_ref(), ) .await?; let mut batch = ReadBatch::from_api(batch); @@ -498,7 +480,7 @@ impl S2Stream { AppendSession::new( self.client.clone(), self.name.clone(), - self.headers(config.stream_config()), + self.encryption.clone(), config, ) } @@ -508,7 +490,7 @@ impl S2Stream { Producer::new( self.client.clone(), self.name.clone(), - self.headers(config.stream_config()), + self.encryption.clone(), config, ) } @@ -522,7 +504,7 @@ impl S2Stream { session::read_session( self.client.clone(), self.name.clone(), - self.headers(input.stream_config.as_ref()), + self.encryption.clone(), input, config, ) diff --git a/sdk/src/producer.rs b/sdk/src/producer.rs index 293f774a..632c31b5 100644 --- a/sdk/src/producer.rs +++ b/sdk/src/producer.rs @@ -20,11 +20,9 @@ use crate::{ api::BasinClient, batching::{AppendInputs, AppendRecordBatches, BatchingConfig}, error::ProducerError, - session::{ - AppendPermit, AppendPermits, AppendSessionInternal, BatchSubmitTicket, StreamHeaders, - }, + session::{AppendPermit, AppendPermits, AppendSessionInternal, BatchSubmitTicket}, types::{ - AppendAck, AppendRecord, FencingToken, MeteredBytes, ONE_MIB, StreamConfig, StreamName, + AppendAck, AppendRecord, EncryptionKey, FencingToken, MeteredBytes, ONE_MIB, StreamName, ValidationError, }, }; @@ -77,7 +75,6 @@ pub struct ProducerConfig { batching: BatchingConfig, fencing_token: Option, match_seq_num: Option, - stream_config: Option, } impl Default for ProducerConfig { @@ -87,7 +84,6 @@ impl Default for ProducerConfig { batching: BatchingConfig::default(), fencing_token: None, match_seq_num: None, - stream_config: None, } } } @@ -141,23 +137,6 @@ impl ProducerConfig { ..self } } - - /// Set the stream configuration to apply if the stream is created on append. - /// - /// Unset fields inherit the basin's default stream configuration. Ignored if the stream - /// already exists. - /// - /// Defaults to `None`. - pub fn with_stream_config(self, stream_config: StreamConfig) -> Self { - Self { - stream_config: Some(stream_config), - ..self - } - } - - pub(crate) fn stream_config(&self) -> Option<&StreamConfig> { - self.stream_config.as_ref() - } } /// High-level interface for submitting individual [`AppendRecord`]s. @@ -175,12 +154,12 @@ impl Producer { pub(crate) fn new( client: BasinClient, stream: StreamName, - headers: StreamHeaders, + encryption: Option, config: ProducerConfig, ) -> Self { let (cmd_tx, cmd_rx) = mpsc::channel::(RECORD_BATCH_MAX.count); let permits = AppendPermits::new(None, config.max_unacked_bytes); - let session = AppendSessionInternal::new(client, stream, headers); + let session = AppendSessionInternal::new(client, stream, encryption); let terminal_err = Arc::new(OnceLock::new()); let _handle = AbortOnDropHandle::new(tokio::spawn(Self::run( session, diff --git a/sdk/src/retry.rs b/sdk/src/retry.rs index 8eddd738..ac7bb370 100644 --- a/sdk/src/retry.rs +++ b/sdk/src/retry.rs @@ -2,33 +2,6 @@ use std::time::Duration; use rand::{RngExt, rng}; -use crate::frame_signal::FrameSignal; - -pub(crate) trait AppendRetryError: Sized { - /// Whether the failure represented by this error guarantees that no mutation occurred. - fn has_no_side_effects(&self) -> bool; - - /// Mark the whole append as indefinite, retaining this final attempt's definite error. - /// The caller must establish that an earlier attempt may have taken effect. - fn into_indefinite_failure(self) -> Self; - - /// Wrap a definite final error if an earlier attempt may have taken effect. - /// Already indefinite errors are returned unchanged. - fn with_prior_uncertainty(self, prior_uncertainty: bool) -> Self { - if prior_uncertainty && self.has_no_side_effects() { - self.into_indefinite_failure() - } else { - self - } - } - - /// Whether this attempt may have taken effect, accounting for unsent request data. - /// Without a frame signal, assume the request may have been sent. - fn attempt_may_have_side_effects(&self, frame_signal: Option<&FrameSignal>) -> bool { - !self.has_no_side_effects() && frame_signal.is_none_or(|s| s.is_signalled()) - } -} - #[derive(Debug, Clone, Copy)] pub struct RetryBackoffBuilder { pub min_base_delay: Duration, diff --git a/sdk/src/session/append.rs b/sdk/src/session/append.rs index 43989a7b..ea1f90fa 100644 --- a/sdk/src/session/append.rs +++ b/sdk/src/session/append.rs @@ -23,11 +23,10 @@ use crate::{ error::{AppendError, RequestError}, frame_signal::FrameSignal, reconnect::{AdvisedReconnects, ReconnectAdvice}, - retry::{AppendRetryError, RetryBackoffBuilder}, - session::StreamHeaders, + retry::RetryBackoffBuilder, types::{ - AccessTokenMode, AppendAck, AppendInput, AppendRetryPolicy, MeteredBytes, ONE_MIB, - StreamConfig, StreamName, StreamPosition, ValidationError, + AccessTokenMode, AppendAck, AppendInput, AppendRetryPolicy, EncryptionKey, MeteredBytes, + ONE_MIB, StreamName, StreamPosition, ValidationError, }, }; @@ -59,16 +58,6 @@ pub enum AppendSessionError { /// The server returned an invalid append acknowledgement. #[error("invalid append acknowledgement: {0}")] InvalidAck(String), - /// The final attempt failed definitively, but an earlier attempt may have taken effect, - /// so the entire append operation is indeterminate. - #[error( - "append may have taken effect in an earlier attempt; final attempt failed: {final_attempt_error}" - )] - IndefiniteFailure { - /// The definite error returned by the final attempt. - #[source] - final_attempt_error: Box, - }, } impl AppendSessionError { @@ -76,9 +65,6 @@ impl AppendSessionError { pub fn is_retryable(&self) -> bool { match self { Self::Append(error) => error.is_retryable(), - Self::IndefiniteFailure { - final_attempt_error, - } => final_attempt_error.is_retryable(), Self::AckTimeout | Self::ServerDisconnected => true, Self::StreamClosedEarly | Self::SessionClosed @@ -92,7 +78,6 @@ impl AppendSessionError { pub fn has_no_side_effects(&self) -> bool { match self { Self::Append(error) => error.has_no_side_effects(), - Self::IndefiniteFailure { .. } => false, Self::SessionClosed | Self::SessionClosing => true, Self::AckTimeout | Self::ServerDisconnected @@ -106,9 +91,6 @@ impl AppendSessionError { pub fn request_error(&self) -> Option<&RequestError> { match self { Self::Append(error) => error.request_error(), - Self::IndefiniteFailure { - final_attempt_error, - } => final_attempt_error.request_error(), Self::AckTimeout | Self::ServerDisconnected | Self::StreamClosedEarly @@ -134,21 +116,14 @@ impl AppendSessionError { } } -impl AppendRetryError for AppendSessionError { - fn has_no_side_effects(&self) -> bool { - Self::has_no_side_effects(self) - } - - fn into_indefinite_failure(self) -> Self { - Self::IndefiniteFailure { - final_attempt_error: Box::new(self), - } - } -} - impl From for AppendSessionError { fn from(error: ApiError) -> Self { - Self::Append(error.into()) + match error { + ApiError::AppendConditionFailed(condition) => { + Self::Append(AppendError::ConditionFailed(condition.into())) + } + other => Self::Append(AppendError::Request(other.into())), + } } } @@ -179,7 +154,6 @@ impl Future for BatchSubmitTicket { pub struct AppendSessionConfig { max_unacked_bytes: u32, max_unacked_batches: Option, - stream_config: Option, } impl Default for AppendSessionConfig { @@ -187,7 +161,6 @@ impl Default for AppendSessionConfig { Self { max_unacked_bytes: 5 * ONE_MIB, max_unacked_batches: None, - stream_config: None, } } } @@ -222,23 +195,6 @@ impl AppendSessionConfig { ..self } } - - /// Set the stream configuration to apply if the stream is created on append. - /// - /// Unset fields inherit the basin's default stream configuration. Ignored if the stream - /// already exists. - /// - /// Defaults to `None`. - pub fn with_stream_config(self, stream_config: StreamConfig) -> Self { - Self { - stream_config: Some(stream_config), - ..self - } - } - - pub(crate) fn stream_config(&self) -> Option<&StreamConfig> { - self.stream_config.as_ref() - } } struct SessionState { @@ -275,7 +231,7 @@ impl AppendSession { pub(crate) fn new( client: BasinClient, stream: StreamName, - headers: StreamHeaders, + encryption: Option, config: AppendSessionConfig, ) -> Self { let buffer_size = config @@ -289,7 +245,7 @@ impl AppendSession { let handle = AbortOnDropHandle::new(tokio::spawn(run_session_with_retry( client, stream, - headers, + encryption, cmd_rx, retry_builder, buffer_size, @@ -400,7 +356,11 @@ pub(crate) struct AppendSessionInternal { } impl AppendSessionInternal { - pub(crate) fn new(client: BasinClient, stream: StreamName, headers: StreamHeaders) -> Self { + pub(crate) fn new( + client: BasinClient, + stream: StreamName, + encryption: Option, + ) -> Self { let buffer_size = DEFAULT_CHANNEL_BUFFER_SIZE; let (cmd_tx, cmd_rx) = mpsc::channel(buffer_size); let retry_builder = retry_builder(&client.config.retry); @@ -408,7 +368,7 @@ impl AppendSessionInternal { let handle = AbortOnDropHandle::new(tokio::spawn(run_session_with_retry( client, stream, - headers, + encryption, cmd_rx, retry_builder, buffer_size, @@ -513,7 +473,7 @@ impl AppendPermits { async fn run_session_with_retry( client: BasinClient, stream: StreamName, - headers: StreamHeaders, + encryption: Option, cmd_rx: mpsc::Receiver, retry_builder: RetryBackoffBuilder, buffer_size: usize, @@ -543,7 +503,7 @@ async fn run_session_with_retry( let result = run_session( &client, &stream, - &headers, + encryption.as_ref(), &mut state, buffer_size, &frame_signal, @@ -588,11 +548,6 @@ async fn run_session_with_retry( access_token_mode, ) && let Some(backoff) = retry_backoff.next() { - if err.attempt_may_have_side_effects(frame_signal.as_ref()) { - for append in &mut state.inflight_appends { - append.prior_uncertainty = true; - } - } debug!( %err, ?backoff, @@ -607,16 +562,12 @@ async fn run_session_with_retry( "not retrying append session" ); - let session_err = err.clone().with_prior_uncertainty( - state.inflight_appends.iter().any(|a| a.prior_uncertainty), - ); - let _ = terminal_err.set(session_err.clone()); + let err: AppendSessionError = err; + + let _ = terminal_err.set(err.clone()); for inflight_append in state.inflight_appends.drain(..) { - let error = err - .clone() - .with_prior_uncertainty(inflight_append.prior_uncertainty); - let _ = inflight_append.ack_tx.send(Err(error)); + let _ = inflight_append.ack_tx.send(Err(err.clone())); } if let Some(stashed) = state.stashed_submission.take() { @@ -624,16 +575,12 @@ async fn run_session_with_retry( } if let Some(done_tx) = state.close_tx.take() { - let _ = done_tx.send(Err(session_err.clone())); + let _ = done_tx.send(Err(err.clone())); } state.cmd_rx.close(); while let Some(cmd) = state.cmd_rx.recv().await { - let error = match &cmd { - Command::Submit { .. } => &err, - Command::Close { .. } => &session_err, - }; - cmd.reject(error.clone()); + cmd.reject(err.clone()); } break; } @@ -657,7 +604,7 @@ enum SessionOutcome { async fn run_session( client: &BasinClient, stream: &StreamName, - headers: &StreamHeaders, + encryption: Option<&EncryptionKey>, state: &mut SessionState, buffer_size: usize, frame_signal: &Option, @@ -671,7 +618,7 @@ async fn run_session( let (input_tx, mut acks) = connect( client, stream, - headers, + encryption, buffer_size, frame_signal.clone(), reconnect.clone(), @@ -741,7 +688,6 @@ async fn run_session( ack_tx: submission.ack_tx, ack_deadline, _permit: submission.permit, - prior_uncertainty: false, }); } @@ -942,7 +888,7 @@ async fn drain_for_reconnect( async fn connect( client: &BasinClient, stream: &StreamName, - headers: &StreamHeaders, + encryption: Option<&EncryptionKey>, buffer_size: usize, frame_signal: Option, reconnect: ReconnectAdvice, @@ -953,8 +899,7 @@ async fn connect( .append_session( stream, ReceiverStream::new(input_rx).map(|i| i.into()), - headers.encryption.as_ref(), - headers.stream_config.as_ref(), + encryption, frame_signal, reconnect, ) @@ -1037,7 +982,6 @@ struct InflightAppend { ack_tx: oneshot::Sender>, ack_deadline: Instant, _permit: Option, - prior_uncertainty: bool, } enum Command { @@ -1074,7 +1018,9 @@ fn is_safe_to_retry( let policy_compliant = match policy { AppendRetryPolicy::All => true, AppendRetryPolicy::NoSideEffects => { - !has_inflight || !err.attempt_may_have_side_effects(frame_signal) + !has_inflight + || !frame_signal.is_none_or(|s| s.is_signalled()) + || err.has_no_side_effects() } }; policy_compliant @@ -1110,16 +1056,13 @@ impl From for TimerEvent { #[cfg(test)] mod tests { - use std::error::Error; - use http::StatusCode; use super::{AppendSessionError, is_safe_to_retry}; use crate::{ api::{ApiError, ServerErrorBody}, - error::{AppendError, ProducerError, RequestError}, + error::{AppendError, RequestError}, frame_signal::FrameSignal, - retry::AppendRetryError, types::{AccessTokenMode, AppendRetryPolicy}, }; @@ -1133,61 +1076,6 @@ mod tests { )))) } - #[rstest::rstest] - #[case(StatusCode::FORBIDDEN, "permission_denied", false, false)] - #[case(StatusCode::FORBIDDEN, "permission_denied", true, true)] - #[case(StatusCode::SERVICE_UNAVAILABLE, "unavailable", false, false)] - #[case(StatusCode::SERVICE_UNAVAILABLE, "unavailable", true, false)] - #[case(StatusCode::TOO_MANY_REQUESTS, "rate_limited", true, true)] - #[test] - fn session_failure_preserves_uncertainty_and_latest_error( - #[case] status: StatusCode, - #[case] code: &str, - #[case] prior_uncertainty: bool, - #[case] wrapped: bool, - ) { - let latest = server_error(status, code); - let error = latest.clone().with_prior_uncertainty(prior_uncertainty); - assert_eq!( - matches!(error, AppendSessionError::IndefiniteFailure { .. }), - wrapped - ); - assert_eq!(error.is_retryable(), latest.is_retryable()); - assert_eq!( - error.has_no_side_effects(), - !wrapped && latest.has_no_side_effects() - ); - assert_eq!( - error.request_error().unwrap().server_error().unwrap().code, - code - ); - if wrapped { - let source = error - .source() - .unwrap() - .downcast_ref::>() - .unwrap(); - assert_eq!(source.to_string(), latest.to_string()); - assert!(source.has_no_side_effects()); - } - - let producer_error = ProducerError::from(error.clone()); - assert_eq!(producer_error.is_retryable(), error.is_retryable()); - assert_eq!( - producer_error.has_no_side_effects(), - error.has_no_side_effects() - ); - assert_eq!( - producer_error - .request_error() - .unwrap() - .server_error() - .unwrap() - .code, - code - ); - } - #[test] fn safe_to_retry_session_all_policy() { let retryable = server_error(StatusCode::INTERNAL_SERVER_ERROR, "internal"); diff --git a/sdk/src/session/mod.rs b/sdk/src/session/mod.rs index 7868ed29..107df6e4 100644 --- a/sdk/src/session/mod.rs +++ b/sdk/src/session/mod.rs @@ -5,12 +5,3 @@ pub(crate) use append::{AppendPermit, AppendPermits, AppendSessionInternal, Batc pub use append::{AppendSession, AppendSessionConfig}; pub(crate) use read::read_session; pub use read::{ReadSession, ReadSessionError}; - -/// Per-stream options sent as request headers on every (re)connect of a session. -#[derive(Debug, Clone, Default)] -pub(crate) struct StreamHeaders { - /// `s2-encryption-key` - pub encryption: Option, - /// `s2-stream-config` - pub stream_config: Option, -} diff --git a/sdk/src/session/read.rs b/sdk/src/session/read.rs index 83f848f4..a3a07e99 100644 --- a/sdk/src/session/read.rs +++ b/sdk/src/session/read.rs @@ -22,9 +22,8 @@ use crate::{ error::{ReadError, RequestError}, reconnect::{AdvisedReconnects, ReconnectAdvice}, retry::RetryBackoff, - session::StreamHeaders, types::{ - AccessTokenMode, MeteredBytes, ReadBatch, ReadInput, ReadSessionConfig, + AccessTokenMode, EncryptionKey, MeteredBytes, ReadBatch, ReadInput, ReadSessionConfig, ReadSessionRetryPolicy, StreamName, StreamPosition, }, }; @@ -372,7 +371,7 @@ impl Drop for ReadSession { pub async fn read_session( client: BasinClient, name: StreamName, - headers: StreamHeaders, + encryption: Option, input: ReadInput, config: ReadSessionConfig, ) -> Result { @@ -380,7 +379,6 @@ pub async fn read_session( start, stop, ignore_command_records, - stream_config: _, } = input; let mut start: ReadStart = start.into(); let mut end: ReadEnd = stop.into(); @@ -401,7 +399,7 @@ pub async fn read_session( match session_inner( client.clone(), name.clone(), - headers.clone(), + encryption.clone(), start.clone(), end.clone(), ReconnectAdvice::default(), @@ -441,7 +439,7 @@ pub async fn read_session( match session_inner( client.clone(), name.clone(), - headers.clone(), + encryption.clone(), start.clone(), end.clone(), ReconnectAdvice::default(), @@ -581,21 +579,14 @@ fn update_resume_start(start: &mut ReadStart, batch: &ReadBatch) { async fn session_inner( client: BasinClient, name: StreamName, - headers: StreamHeaders, + encryption: Option, start: ReadStart, end: ReadEnd, reconnect: ReconnectAdvice, advised_reconnects: AdvisedReconnects, ) -> Result, ReadSessionFailure> { let mut batches = client - .read_session( - &name, - start, - end, - headers.encryption.as_ref(), - headers.stream_config.as_ref(), - reconnect.clone(), - ) + .read_session(&name, start, end, encryption.as_ref(), reconnect.clone()) .await?; let mut declined_advice = false; diff --git a/sdk/src/types.rs b/sdk/src/types.rs index 099048d2..6d324782 100644 --- a/sdk/src/types.rs +++ b/sdk/src/types.rs @@ -15,9 +15,7 @@ use std::{ #[cfg(feature = "_hidden")] use async_trait::async_trait; use bytes::Bytes; -use compact_str::CompactString; use http::{ - HeaderMap, header::HeaderValue, uri::{Authority, Scheme}, }; @@ -513,7 +511,6 @@ pub struct S2Config { pub(crate) retry: RetryConfig, pub(crate) compression: Compression, pub(crate) user_agent: HeaderValue, - pub(crate) default_headers: HeaderMap, pub(crate) insecure_skip_cert_verification: bool, pub(crate) rustls_crypto_provider: Option>, } @@ -531,7 +528,6 @@ impl S2Config { user_agent: concat!("s2-sdk-rust/", env!("CARGO_PKG_VERSION")) .parse() .expect("valid user agent"), - default_headers: HeaderMap::new(), insecure_skip_cert_verification: false, rustls_crypto_provider: default_rustls_crypto_provider(), } @@ -551,54 +547,6 @@ impl S2Config { Self { endpoints, ..self } } - /// Set additional HTTP headers to send with every request. - /// - /// These headers apply to account, basin, and stream operations, including - /// retries and streaming requests. SDK-generated headers, such as - /// authorization and basin routing, take precedence over these defaults. - /// Calling this method again replaces the previous set of default headers. - /// - /// `Accept-Encoding` defaults are used only when [`Compression::None`] is - /// configured; otherwise the SDK sets the header to the configured - /// compression algorithm. - /// - /// Headers are sent to all configured S2 endpoints. Use - /// [`HeaderValue::set_sensitive`] for values that should be redacted in debug - /// output. Do not use these defaults for per-request identifiers, since the - /// same values are reused across requests. - /// - /// # Errors - /// - /// Returns an error if `default_headers` contains `Content-Type`, - /// `Content-Encoding`, `Content-Length`, or `Transfer-Encoding`. - /// The SDK controls request format and body framing. - /// Use [`Self::with_compression`] to configure request body encoding. - #[cfg(feature = "_hidden")] - #[doc(hidden)] - pub fn with_default_headers(self, default_headers: HeaderMap) -> Result { - if default_headers.contains_key(http::header::CONTENT_ENCODING) { - return Err(ValidationError( - "Content-Encoding cannot be set in default headers; use S2Config::with_compression instead" - .into(), - )); - } - for name in [ - http::header::CONTENT_TYPE, - http::header::CONTENT_LENGTH, - http::header::TRANSFER_ENCODING, - ] { - if default_headers.contains_key(&name) { - return Err(ValidationError(format!( - "{name} cannot be set in default headers; the SDK controls request format and body framing" - ))); - } - } - Ok(Self { - default_headers, - ..self - }) - } - /// Set the timeout for establishing a connection to the server. /// /// Defaults to `3s`. @@ -734,6 +682,33 @@ impl Page { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Storage class for recent appends. +pub enum StorageClass { + /// Standard storage class that offers append latencies under `500ms`. + Standard, + /// Express storage class that offers append latencies under `50ms`. + Express, +} + +impl From for StorageClass { + fn from(value: api::config::StorageClass) -> Self { + match value { + api::config::StorageClass::Standard => StorageClass::Standard, + api::config::StorageClass::Express => StorageClass::Express, + } + } +} + +impl From for api::config::StorageClass { + fn from(value: StorageClass) -> Self { + match value { + StorageClass::Standard => api::config::StorageClass::Standard, + StorageClass::Express => api::config::StorageClass::Express, + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] /// Retention policy for records in a stream. pub enum RetentionPolicy { @@ -893,8 +868,10 @@ impl From for api::config::DeleteOnEmptyConfig { #[non_exhaustive] /// Configuration for a stream. pub struct StreamConfig { - /// [Storage class](https://s2.dev/docs/storage-classes) for the stream. - pub storage_class: Option, + /// Storage class for the stream. + /// + /// Defaults to [`Express`](StorageClass::Express). + pub storage_class: Option, /// Retention policy for records in the stream. /// /// Defaults to `7 days` of retention. @@ -916,9 +893,9 @@ impl StreamConfig { } /// Set the storage class for the stream. - pub fn with_storage_class(self, storage_class: impl Into) -> Self { + pub fn with_storage_class(self, storage_class: StorageClass) -> Self { Self { - storage_class: Some(storage_class.into()), + storage_class: Some(storage_class), ..self } } @@ -951,7 +928,7 @@ impl StreamConfig { impl From for StreamConfig { fn from(value: api::config::StreamConfig) -> Self { Self { - storage_class: value.storage_class, + storage_class: value.storage_class.map(Into::into), retention_policy: value.retention_policy.map(Into::into), timestamping: value.timestamping.map(Into::into), delete_on_empty: value.delete_on_empty.map(Into::into), @@ -962,7 +939,7 @@ impl From for StreamConfig { impl From for api::config::StreamConfig { fn from(value: StreamConfig) -> Self { Self { - storage_class: value.storage_class, + storage_class: value.storage_class.map(Into::into), retention_policy: value.retention_policy.map(Into::into), timestamping: value.timestamping.map(Into::into), delete_on_empty: value.delete_on_empty.map(Into::into), @@ -1440,7 +1417,7 @@ impl From for api::config::DeleteOnEmptyReconfigur /// Reconfiguration for [`StreamConfig`]. pub struct StreamReconfiguration { /// Override for the existing [`storage_class`](StreamConfig::storage_class). - pub storage_class: Maybe>, + pub storage_class: Maybe>, /// Override for the existing [`retention_policy`](StreamConfig::retention_policy). pub retention_policy: Maybe>, /// Override for the existing [`timestamping`](StreamConfig::timestamping). @@ -1456,9 +1433,9 @@ impl StreamReconfiguration { } /// Set the override for the existing [`storage_class`](StreamConfig::storage_class). - pub fn with_storage_class(self, storage_class: impl Into) -> Self { + pub fn with_storage_class(self, storage_class: StorageClass) -> Self { Self { - storage_class: Maybe::Specified(Some(storage_class.into())), + storage_class: Maybe::Specified(Some(storage_class)), ..self } } @@ -1491,7 +1468,7 @@ impl StreamReconfiguration { impl From for api::config::StreamReconfiguration { fn from(value: StreamReconfiguration) -> Self { Self { - storage_class: value.storage_class, + storage_class: value.storage_class.map(|m| m.map(Into::into)), retention_policy: value.retention_policy.map(|m| m.map(Into::into)), timestamping: value.timestamping.map(|m| m.map(Into::into)), delete_on_empty: value.delete_on_empty.map(|m| m.map(Into::into)), @@ -1683,10 +1660,6 @@ pub struct LocationInfo { pub name: LocationName, /// Location represents a private placement, limited by account. pub is_private: bool, - /// [Storage classes](https://s2.dev/docs/storage-classes) available to the account in this location. - pub storage_classes: Option>, - /// Default [storage class](https://s2.dev/docs/storage-classes) for this location. - pub default_storage_class: Option, } impl From for LocationInfo { @@ -1694,8 +1667,6 @@ impl From for LocationInfo { Self { name: value.name, is_private: value.is_private, - storage_classes: value.storage_classes, - default_storage_class: value.default_storage_class, } } } @@ -2433,7 +2404,7 @@ pub enum BasinMetricSet { /// Returns a [`GaugeMetric`] representing a timeseries of total stored bytes across all streams /// in the basin, with one observed value for each hour over the requested time range. Storage(TimeRange), - /// Returns [`AccumulationMetric`]s, one per storage class. + /// Returns [`AccumulationMetric`]s, one per storage class (standard, express). /// /// Each metric represents a timeseries of the number of append operations across all streams /// in the basin, with one accumulated value per interval over the requested time range. @@ -3391,16 +3362,6 @@ pub struct AppendInput { /// If unspecified, no matching is performed. If specified and mismatched, /// the append fails. A stream defaults to `""` as its fencing token. pub fencing_token: Option, - /// Stream configuration to apply if the stream is created on append. - /// - /// Unset fields inherit the basin's default stream configuration. Ignored if the stream - /// already exists. - /// - /// Only used by [`append`](crate::S2Stream::append). Append sessions send the header once - /// at connect; see - /// [`AppendSessionConfig::with_stream_config`](crate::append_session::AppendSessionConfig::with_stream_config) - /// and [`ProducerConfig::with_stream_config`](crate::producer::ProducerConfig::with_stream_config). - pub stream_config: Option, } impl AppendInput { @@ -3410,15 +3371,6 @@ impl AppendInput { records, match_seq_num: None, fencing_token: None, - stream_config: None, - } - } - - /// Set the stream configuration to apply if the stream is created on append. - pub fn with_stream_config(self, stream_config: StreamConfig) -> Self { - Self { - stream_config: Some(stream_config), - ..self } } @@ -3675,11 +3627,6 @@ pub struct ReadInput { /// /// Defaults to `false`. pub ignore_command_records: bool, - /// Stream configuration to apply if the stream is created on read. - /// - /// Unset fields inherit the basin's default stream configuration. Ignored if the stream - /// already exists. - pub stream_config: Option, } #[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] @@ -3747,14 +3694,6 @@ impl ReadInput { ..self } } - - /// Set the stream configuration to apply if the stream is created on read. - pub fn with_stream_config(self, stream_config: StreamConfig) -> Self { - Self { - stream_config: Some(stream_config), - ..self - } - } } #[derive(Debug, Clone)] @@ -4050,51 +3989,15 @@ mod tests { assert!(!cfg.insecure_skip_cert_verification); } - #[cfg(feature = "_hidden")] - #[rstest] - #[case::matching_compression("content-encoding", "gzip", Compression::Gzip)] - #[case::mixed_case("Content-Encoding", "identity", Compression::None)] - #[case::empty_value("content-encoding", "", Compression::None)] - fn default_headers_reject_content_encoding( - #[case] name: &str, - #[case] value: &str, - #[case] compression: Compression, - ) { - let headers = HeaderMap::from_iter([( - name.parse::().unwrap(), - HeaderValue::from_str(value).unwrap(), - )]); - let error = S2Config::new("token") - .with_compression(compression) - .with_default_headers(headers) - .unwrap_err(); - assert!(error.0.contains("Content-Encoding")); - assert!(error.0.contains("with_compression")); - } + // -- StorageClass -- - #[cfg(feature = "_hidden")] #[rstest] - #[case::content_type_s2s("content-type", "s2s/proto")] - #[case::content_type_protobuf("content-type", "application/protobuf")] - #[case::content_type_json("content-type", "application/json")] - #[case::content_type_mixed_case("Content-Type", "s2s/proto")] - #[case::content_type_empty("content-type", "")] - #[case::content_length("content-length", "123")] - #[case::content_length_mixed_case("Content-Length", "0")] - #[case::content_length_empty("content-length", "")] - #[case::transfer_encoding("transfer-encoding", "chunked")] - #[case::transfer_encoding_mixed_case("Transfer-Encoding", "chunked")] - #[case::transfer_encoding_empty("transfer-encoding", "")] - fn default_headers_reject_framing_headers(#[case] name: &str, #[case] value: &str) { - let headers = HeaderMap::from_iter([( - name.parse::().unwrap(), - HeaderValue::from_str(value).unwrap(), - )]); - let error = S2Config::new("token") - .with_default_headers(headers) - .unwrap_err(); - assert!(error.0.contains(&name.to_ascii_lowercase())); - assert!(error.0.contains("framing")); + #[case::standard(StorageClass::Standard)] + #[case::express(StorageClass::Express)] + fn storage_class_roundtrip(#[case] sdk: StorageClass) { + let api: api::config::StorageClass = sdk.into(); + let back: StorageClass = api.into(); + assert_eq!(back, sdk); } // -- RetentionPolicy -- @@ -4158,7 +4061,7 @@ mod tests { #[test] fn stream_config_builder_and_roundtrip() { let sdk = StreamConfig::new() - .with_storage_class("express") + .with_storage_class(StorageClass::Express) .with_retention_policy(RetentionPolicy::Age(86400)) .with_timestamping(TimestampingConfig { mode: Some(TimestampingMode::ClientPrefer), @@ -4175,7 +4078,9 @@ mod tests { #[test] fn basin_config_builder_and_roundtrip() { let sdk = BasinConfig::new() - .with_default_stream_config(StreamConfig::new().with_storage_class("standard")) + .with_default_stream_config( + StreamConfig::new().with_storage_class(StorageClass::Standard), + ) .with_create_stream_on_append(true) .with_create_stream_on_read(false); let api: api::config::BasinConfig = sdk.clone().into(); diff --git a/sdk/tests/account_ops.rs b/sdk/tests/account_ops.rs index 4576a36a..552445d7 100644 --- a/sdk/tests/account_ops.rs +++ b/sdk/tests/account_ops.rs @@ -70,7 +70,7 @@ async fn basin_config_roundtrip() -> Result<(), Box> { let config = BasinConfig::new() .with_default_stream_config( StreamConfig::new() - .with_storage_class("express") + .with_storage_class(StorageClass::Express) .with_delete_on_empty( DeleteOnEmptyConfig::new().with_min_age(Duration::from_secs(60)), ), @@ -85,7 +85,7 @@ async fn basin_config_roundtrip() -> Result<(), Box> { retrieved_config, BasinConfig { default_stream_config: Some(StreamConfig { - storage_class: Some(ref storage_class), + storage_class: Some(StorageClass::Express), delete_on_empty: Some(DeleteOnEmptyConfig { min_age_secs: 60, .. @@ -94,7 +94,7 @@ async fn basin_config_roundtrip() -> Result<(), Box> { }), create_stream_on_read: true, .. - } if storage_class == "express" + } ); s2.delete_basin(DeleteBasinInput::new(basin_name)).await?; @@ -114,7 +114,9 @@ async fn reconfigure_basin() -> Result<(), Box> { .await?; let new_config = BasinReconfiguration::new() - .with_default_stream_config(StreamReconfiguration::new().with_storage_class("standard")) + .with_default_stream_config( + StreamReconfiguration::new().with_storage_class(StorageClass::Standard), + ) .with_create_stream_on_append(false); let updated_config = s2 @@ -125,12 +127,12 @@ async fn reconfigure_basin() -> Result<(), Box> { updated_config, BasinConfig { default_stream_config: Some(StreamConfig { - storage_class: Some(ref storage_class), + storage_class: Some(StorageClass::Standard), .. }), create_stream_on_append: false, .. - } if storage_class == "standard" + } ); s2.delete_basin(DeleteBasinInput::new(basin_name)).await?; @@ -337,7 +339,7 @@ async fn get_basin_config() -> Result<(), Box> { let basin_name = unique_basin_name(); let config = BasinConfig::new() - .with_default_stream_config(StreamConfig::new().with_storage_class("express")); + .with_default_stream_config(StreamConfig::new().with_storage_class(StorageClass::Express)); s2.create_basin(CreateBasinInput::new(basin_name.clone()).with_config(config)) .await?; @@ -347,9 +349,9 @@ async fn get_basin_config() -> Result<(), Box> { assert_matches!( retrieved_config.default_stream_config, Some(StreamConfig { - storage_class: Some(ref storage_class), + storage_class: Some(StorageClass::Express), .. - }) if storage_class == "express" + }) ); s2.delete_basin(DeleteBasinInput::new(basin_name)).await?; diff --git a/sdk/tests/basin_ops.rs b/sdk/tests/basin_ops.rs index fc9b2cc4..39506e6b 100644 --- a/sdk/tests/basin_ops.rs +++ b/sdk/tests/basin_ops.rs @@ -55,7 +55,7 @@ async fn create_list_and_delete_stream(basin: &S2Basin) -> Result<(), RequestErr async fn stream_config_roundtrip(basin: &S2Basin) -> Result<(), RequestError> { let stream_name = unique_stream_name(); let config = StreamConfig::new() - .with_storage_class("standard") + .with_storage_class(StorageClass::Standard) .with_retention_policy(RetentionPolicy::Age(3600)) .with_timestamping(TimestampingConfig::new().with_mode(TimestampingMode::ClientRequire)); @@ -68,14 +68,14 @@ async fn stream_config_roundtrip(basin: &S2Basin) -> Result<(), RequestError> { assert_matches!( retrieved_config, StreamConfig { - storage_class: Some(ref storage_class), + storage_class: Some(StorageClass::Standard), retention_policy: Some(RetentionPolicy::Age(3600)), timestamping: Some(TimestampingConfig { mode: Some(TimestampingMode::ClientRequire), .. }), .. - } if storage_class == "standard" + } ); Ok(()) @@ -348,7 +348,7 @@ async fn delete_nonexistent_stream_with_ignore(basin: &S2Basin) -> Result<(), Re async fn get_stream_config(basin: &S2Basin) -> Result<(), RequestError> { let stream_name = unique_stream_name(); - let config = StreamConfig::new().with_storage_class("express"); + let config = StreamConfig::new().with_storage_class(StorageClass::Express); basin .create_stream(CreateStreamInput::new(stream_name.clone()).with_config(config)) @@ -356,7 +356,7 @@ async fn get_stream_config(basin: &S2Basin) -> Result<(), RequestError> { let retrieved_config = basin.get_stream_config(stream_name.clone()).await?; - assert_matches!(retrieved_config.storage_class.as_deref(), Some("express")); + assert_matches!(retrieved_config.storage_class, Some(StorageClass::Express)); Ok(()) } @@ -605,7 +605,7 @@ async fn list_all_streams_include_deleted( async fn create_stream_with_full_config(basin: &SharedS2Basin) -> Result<(), RequestError> { let stream_name = unique_stream_name(); let config = StreamConfig::new() - .with_storage_class("standard") + .with_storage_class(StorageClass::Standard) .with_retention_policy(RetentionPolicy::Age(86400)) .with_timestamping( TimestampingConfig::new() @@ -623,7 +623,7 @@ async fn create_stream_with_full_config(basin: &SharedS2Basin) -> Result<(), Req assert_matches!( retrieved, StreamConfig { - storage_class: Some(ref storage_class), + storage_class: Some(StorageClass::Standard), retention_policy: Some(RetentionPolicy::Age(86400)), timestamping: Some(TimestampingConfig { mode: Some(TimestampingMode::ClientRequire), @@ -635,7 +635,7 @@ async fn create_stream_with_full_config(basin: &SharedS2Basin) -> Result<(), Req .. }), .. - } if storage_class == "standard" + } ); basin @@ -649,7 +649,7 @@ async fn create_stream_with_full_config(basin: &SharedS2Basin) -> Result<(), Req #[tokio_shared_rt::test(shared)] async fn create_stream_storage_class_express(basin: &SharedS2Basin) -> Result<(), RequestError> { let stream_name = unique_stream_name(); - let config = StreamConfig::new().with_storage_class("express"); + let config = StreamConfig::new().with_storage_class(StorageClass::Express); let result = basin .create_stream(CreateStreamInput::new(stream_name.clone()).with_config(config)) @@ -664,7 +664,7 @@ async fn create_stream_storage_class_express(basin: &SharedS2Basin) -> Result<() assert_eq!(info.name, stream_name); let retrieved = basin.get_stream_config(stream_name.clone()).await?; - assert_matches!(retrieved.storage_class.as_deref(), Some("express") | None); + assert_matches!(retrieved.storage_class, Some(StorageClass::Express) | None); basin .delete_stream(DeleteStreamInput::new(stream_name)) @@ -853,11 +853,11 @@ async fn reconfigure_stream_storage_class_standard( let config = basin .reconfigure_stream(ReconfigureStreamInput::new( stream_name.clone(), - StreamReconfiguration::new().with_storage_class("standard"), + StreamReconfiguration::new().with_storage_class(StorageClass::Standard), )) .await?; - assert_matches!(config.storage_class.as_deref(), Some("standard")); + assert_matches!(config.storage_class, Some(StorageClass::Standard)); basin .delete_stream(DeleteStreamInput::new(stream_name)) @@ -879,7 +879,7 @@ async fn reconfigure_stream_storage_class_express( let result = basin .reconfigure_stream(ReconfigureStreamInput::new( stream_name.clone(), - StreamReconfiguration::new().with_storage_class("express"), + StreamReconfiguration::new().with_storage_class(StorageClass::Express), )) .await; @@ -894,7 +894,7 @@ async fn reconfigure_stream_storage_class_express( Err(err) => return Err(err), }; - assert_matches!(config.storage_class.as_deref(), Some("express") | None); + assert_matches!(config.storage_class, Some(StorageClass::Express) | None); basin .delete_stream(DeleteStreamInput::new(stream_name)) @@ -1126,7 +1126,7 @@ async fn reconfigure_stream_empty_config_no_change( .create_stream( CreateStreamInput::new(stream_name.clone()).with_config( StreamConfig::new() - .with_storage_class("standard") + .with_storage_class(StorageClass::Standard) .with_retention_policy(RetentionPolicy::Age(3600)), ), ) @@ -1140,7 +1140,7 @@ async fn reconfigure_stream_empty_config_no_change( .await?; let retrieved = basin.get_stream_config(stream_name.clone()).await?; - assert_matches!(retrieved.storage_class.as_deref(), Some("standard")); + assert_matches!(retrieved.storage_class, Some(StorageClass::Standard)); assert_matches!(retrieved.retention_policy, Some(RetentionPolicy::Age(3600))); basin @@ -1229,7 +1229,7 @@ async fn reconfigure_stream_nonexistent_errors(basin: &SharedS2Basin) -> Result< let result = basin .reconfigure_stream(ReconfigureStreamInput::new( unique_stream_name(), - StreamReconfiguration::new().with_storage_class("standard"), + StreamReconfiguration::new().with_storage_class(StorageClass::Standard), )) .await; diff --git a/sdk/tests/stream_ops.rs b/sdk/tests/stream_ops.rs index feb7cad7..54d81180 100644 --- a/sdk/tests/stream_ops.rs +++ b/sdk/tests/stream_ops.rs @@ -3,7 +3,7 @@ mod common; use std::time::Duration; use assert_matches::assert_matches; -use common::{S2Stream, SharedS2Basin, s2, s2_config, unique_basin_name, unique_stream_name}; +use common::{S2Stream, SharedS2Basin, s2_config, unique_basin_name, unique_stream_name}; use futures_util::{StreamExt, poll}; use rstest::rstest; use s2_sdk::{ @@ -1971,7 +1971,7 @@ async fn create_stream_inherits_basin_default_config() -> Result<(), Box Result<(), Box Result<(), Box> { - let s2 = s2(); - let basin_name = unique_basin_name(); - s2.create_basin( - CreateBasinInput::new(basin_name.clone()).with_config( - BasinConfig::new() - .with_create_stream_on_append(true) - .with_default_stream_config(StreamConfig::new().with_storage_class("standard")), - ), - ) - .await?; - let basin = s2.basin(basin_name.clone()); - - let stream_config = StreamConfig::new() - .with_retention_policy(RetentionPolicy::Age(3600)) - .with_delete_on_empty(DeleteOnEmptyConfig::new().with_min_age(Duration::from_secs(300))); - - let unary_stream = unique_stream_name(); - basin - .stream(unary_stream.clone()) - .append( - AppendInput::new(AppendRecordBatch::try_from_iter([AppendRecord::new( - "hello", - )?])?) - .with_stream_config(stream_config.clone()), - ) - .await?; - let config = basin.get_stream_config(unary_stream).await?; - assert_matches!( - config, - StreamConfig { - storage_class: Some(ref storage_class), - retention_policy: Some(RetentionPolicy::Age(3600)), - delete_on_empty: Some(DeleteOnEmptyConfig { - min_age_secs: 300, - .. - }), - .. - } if storage_class == "standard" - ); - - let session_stream = unique_stream_name(); - let producer = basin - .stream(session_stream.clone()) - .producer(ProducerConfig::new().with_stream_config(stream_config.clone())); - producer.submit(AppendRecord::new("hello")?).await?.await?; - producer.close().await?; - let config = basin.get_stream_config(session_stream).await?; - assert_matches!( - config, - StreamConfig { - retention_policy: Some(RetentionPolicy::Age(3600)), - delete_on_empty: Some(DeleteOnEmptyConfig { - min_age_secs: 300, - .. - }), - .. - } - ); - - let existing_stream = unique_stream_name(); - basin - .create_stream(CreateStreamInput::new(existing_stream.clone())) - .await?; - let before = basin.get_stream_config(existing_stream.clone()).await?; - basin - .stream(existing_stream.clone()) - .append( - AppendInput::new(AppendRecordBatch::try_from_iter([AppendRecord::new( - "hello", - )?])?) - .with_stream_config(stream_config), - ) - .await?; - let after = basin.get_stream_config(existing_stream).await?; - assert_eq!(after, before); - assert_ne!(after.retention_policy, Some(RetentionPolicy::Age(3600))); - - s2.delete_basin(DeleteBasinInput::new(basin_name)).await?; - Ok(()) -} - -#[tokio::test] -async fn stream_config_applies_when_read_creates_stream() -> Result<(), Box> -{ - let s2 = s2(); - let basin_name = unique_basin_name(); - s2.create_basin( - CreateBasinInput::new(basin_name.clone()) - .with_config(BasinConfig::new().with_create_stream_on_read(true)), - ) - .await?; - let basin = s2.basin(basin_name.clone()); - - let stream_config = StreamConfig::new().with_retention_policy(RetentionPolicy::Age(3600)); - - let session_stream = unique_stream_name(); - let _session = basin - .stream(session_stream.clone()) - .read_session( - ReadInput::new().with_stream_config(stream_config), - ReadSessionConfig::default(), - ) - .await?; - let config = basin.get_stream_config(session_stream).await?; - assert_eq!(config.retention_policy, Some(RetentionPolicy::Age(3600))); - - s2.delete_basin(DeleteBasinInput::new(basin_name)).await?; - Ok(()) -} diff --git a/sim/Cargo.lock b/sim/Cargo.lock index 19ae3e7f..a2cf1f39 100644 --- a/sim/Cargo.lock +++ b/sim/Cargo.lock @@ -20,9 +20,9 @@ dependencies = [ [[package]] name = "aegis" -version = "0.9.15" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58541132f980da31e9aa99f7bdee69bc84bf1e168b9b91ef2dbe8abb7b4ce5dd" +checksum = "e07d39d15384924b35b70d7b8fa1f9a2934101dd3fa4722ede163cc4f9b7b960" dependencies = [ "cc", "softaes", @@ -30,36 +30,48 @@ dependencies = [ [[package]] name = "aes" -version = "0.9.3" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" +checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138" dependencies = [ "cipher", "cpubits", - "cpufeatures 0.3.1", - "zeroize", + "cpufeatures 0.3.0", ] [[package]] name = "aes-gcm" -version = "0.11.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f" +checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" dependencies = [ "aead", "aes", "cipher", "ctr", - "ctutils", "ghash", + "subtle", "zeroize", ] [[package]] -name = "aho-corasick" -version = "1.1.5" +name = "ahash" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -78,9 +90,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" dependencies = [ "libc", ] @@ -121,7 +133,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -132,7 +144,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -145,7 +157,7 @@ dependencies = [ "libloading", "linkme", "once_cell", - "rand 0.8.8", + "rand 0.8.6", "rand_core 0.6.4", "rustc_version_runtime", "serde", @@ -154,24 +166,30 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.104" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "arc-swap" -version = "1.9.2" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" dependencies = [ "rustversion", ] [[package]] -name = "arrayvec" -version = "0.7.8" +name = "arrayref" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "asn1-rs" @@ -185,7 +203,7 @@ dependencies = [ "nom 7.1.3", "num-traits", "rusticata-macros", - "thiserror 2.0.20", + "thiserror 2.0.18", "time", ] @@ -197,7 +215,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", "synstructure", ] @@ -209,7 +227,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -226,9 +244,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.46" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f10dafd0c8d2e51ae9a748805777613ed0bbe17bf586b76c8311f45c020a32f" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" dependencies = [ "compression-codecs", "compression-core", @@ -255,26 +273,20 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "async-trait" -version = "0.1.92" +version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] -[[package]] -name = "asyncband" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2d85fd3d291fabcc40c7232c92c280ec1754fd7b5d7ea769f143222143e179a" - [[package]] name = "atoi" version = "3.1.0" @@ -307,9 +319,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.12.0" +version = "1.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" +checksum = "11493b0bad143270fb8ad284a096dd529ba91924c5409adeac856cc1bf047dbc" dependencies = [ "aws-credential-types", "aws-runtime", @@ -321,14 +333,13 @@ dependencies = [ "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", "fastrand", "hex", - "http 1.5.0", - "sha1 0.10.7", + "http 1.4.2", + "sha1 0.10.6", "time", "tokio", "tracing", @@ -338,9 +349,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.3.0" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" +checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -350,9 +361,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.18.1" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -361,22 +372,21 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.45.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" dependencies = [ "cc", "cmake", "dunce", "fs_extra", - "pkg-config", ] [[package]] name = "aws-runtime" -version = "1.9.2" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef47857a1d4488b528f4a5d5715fa7c3300820897824152234d3fa22b1426657" +checksum = "5fc0651c57e384202e47153c1260b84a9936e19803d747615edf199dc3b98d17" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -389,8 +399,8 @@ dependencies = [ "bytes", "bytes-utils", "fastrand", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "percent-encoding", "pin-project-lite", "tracing", @@ -399,11 +409,10 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.109.0" +version = "1.97.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3cfe74df5d9ad2fedd691973ad3521ebf4f27a3c68c792556686aedb5519bab" +checksum = "9aadc669e184501caaa6beafb28c6267fc1baef0810fb58f9b205485ca3f2567" dependencies = [ - "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", @@ -412,24 +421,22 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", "fastrand", "http 0.2.12", - "http 1.5.0", + "http 1.4.2", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-ssooidc" -version = "1.111.0" +version = "1.99.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b0ec31ed6191bd11350aae4b2004198f2db21350cb0a20c57e0a92e55dd161" +checksum = "1342a7db8f358d3de0aed2007a0b54e875458e39848d54cc1d46700b2bfcb0a8" dependencies = [ - "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", @@ -438,24 +445,22 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", "fastrand", "http 0.2.12", - "http 1.5.0", + "http 1.4.2", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-sts" -version = "1.114.0" +version = "1.102.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef45745026107ec30c4ef86bd8ae4b002e7e5f6a86e4225240bdf6b06a0b944a" +checksum = "0fc35b7a14cabdad13795fbbbd26d5ddec0882c01492ceedf2af575aad5f37dd" dependencies = [ - "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", @@ -465,22 +470,21 @@ dependencies = [ "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", "aws-smithy-types", "aws-smithy-xml", "aws-types", "fastrand", "http 0.2.12", - "http 1.5.0", + "http 1.4.2", "regex-lite", "tracing", ] [[package]] name = "aws-sigv4" -version = "1.5.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f" +checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -489,20 +493,20 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.12.1", "http 0.2.12", - "http 1.5.0", + "http 1.4.2", "percent-encoding", - "sha2", + "sha2 0.10.9", "time", "tracing", ] [[package]] name = "aws-smithy-async" -version = "1.3.0" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" +checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" dependencies = [ "futures-util", "pin-project-lite", @@ -511,9 +515,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.64.0" +version = "0.63.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" +checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -521,8 +525,8 @@ dependencies = [ "bytes-utils", "futures-core", "futures-util", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "percent-encoding", "pin-project-lite", @@ -532,15 +536,15 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.4.0" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebfd138fac0337cee7516c352757ea73b9f2266e57d0bcb5bc70e9547e45aef1" +checksum = "5c3ef8931ad1c98aa6a55b4256f847f3116090819844e0dd41ea682cac5dd2d3" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", "h2", - "http 1.5.0", + "http 1.4.2", "hyper", "hyper-rustls", "hyper-util", @@ -556,9 +560,9 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.63.0" +version = "0.62.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dc65a121adb4b33729919fcfa14fa36fb33c1555a8f06bb0e2188dbfdc1d9ef" +checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-schema", @@ -567,31 +571,28 @@ dependencies = [ [[package]] name = "aws-smithy-observability" -version = "0.3.0" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" +checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-query" -version = "0.62.0" +version = "0.60.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62" +checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" dependencies = [ - "aws-smithy-runtime-api", - "aws-smithy-schema", "aws-smithy-types", - "aws-smithy-xml", "urlencoding", ] [[package]] name = "aws-smithy-runtime" -version = "1.14.0" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b82e438d30e02a825d363bd639a9efaed68a8089d86101054b0081e7e0d3e606" +checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -603,9 +604,9 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "http 1.5.0", + "http 1.4.2", "http-body 0.4.6", - "http-body 1.1.0", + "http-body 1.0.1", "http-body-util", "pin-project-lite", "pin-utils", @@ -615,16 +616,16 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.16.0" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223" +checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.5.0", + "http 1.4.2", "pin-project-lite", "tokio", "tracing", @@ -633,39 +634,39 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api-macros" -version = "1.1.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" +checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "aws-smithy-schema" -version = "0.2.0" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" +checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", - "http 1.5.0", + "http 1.4.2", ] [[package]] name = "aws-smithy-types" -version = "1.6.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9" +checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85" dependencies = [ "base64-simd", "bytes", "bytes-utils", "http 0.2.12", - "http 1.5.0", + "http 1.4.2", "http-body 0.4.6", - "http-body 1.1.0", + "http-body 1.0.1", "http-body-util", "itoa", "num-integer", @@ -678,21 +679,18 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.62.0" +version = "0.60.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8" +checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" dependencies = [ - "aws-smithy-runtime-api", - "aws-smithy-schema", - "aws-smithy-types", "xmlparser", ] [[package]] name = "aws-types" -version = "1.6.0" +version = "1.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" +checksum = "d16bf10b03a3c01e6b3b7d47cd964e873ffe9e7d4e80fad16bd4c077cb068531" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -714,8 +712,8 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-util", @@ -745,8 +743,8 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -764,7 +762,7 @@ checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -777,8 +775,8 @@ dependencies = [ "bytes", "either", "fs-err", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "hyper", "hyper-util", "pin-project-lite", @@ -806,12 +804,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" - [[package]] name = "base64-simd" version = "0.8.0" @@ -848,21 +840,22 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.13.2" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "blake3" -version = "1.8.7" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" dependencies = [ + "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", ] [[package]] @@ -891,15 +884,15 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytemuck" -version = "1.25.2" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" [[package]] name = "bytes" -version = "1.12.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" dependencies = [ "serde", ] @@ -916,9 +909,9 @@ dependencies = [ [[package]] name = "bytesize" -version = "2.7.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7354288c522e7e980fafd2075d63d1285794c3a6a16cdd492f189ea406e5f18b" +checksum = "351a3e803ee3c6eaeee6b00076b767514b37c32a73d326c3ec7abddb7d6c3493" [[package]] name = "bytestring" @@ -940,9 +933,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.5" +version = "1.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" +checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" dependencies = [ "find-msvc-tools", "jobserver", @@ -958,18 +951,18 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chacha20" -version = "0.10.2" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", "rand_core 0.10.1", ] @@ -1000,9 +993,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.6" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -1010,9 +1003,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.6" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", @@ -1022,14 +1015,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.4" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] @@ -1053,6 +1046,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "cmsketch" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7ee2cfacbd29706479902b06d75ad8f1362900836aa32799eabc7e004bfd854" + [[package]] name = "colorchoice" version = "1.0.5" @@ -1061,9 +1060,9 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "combine" -version = "4.6.8" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ "bytes", "memchr", @@ -1085,22 +1084,22 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.41" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58a6d0db8759036a783bc7c3f7a07f8cef3bf9470eb1db3bc86e8bcd1c5d0fe8" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" dependencies = [ "compression-core", "flate2", "memchr", - "zstd 0.14.0", - "zstd-safe 8.0.0", + "zstd", + "zstd-safe", ] [[package]] name = "compression-core" -version = "0.4.33" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" [[package]] name = "concurrent-queue" @@ -1167,9 +1166,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] @@ -1186,18 +1185,18 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-epoch" -version = "0.9.21" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ "crossbeam-utils", ] @@ -1214,9 +1213,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.23" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crypto-common" @@ -1277,7 +1276,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -1288,7 +1287,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -1307,15 +1306,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.11.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" - -[[package]] -name = "datasketches" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "der-parser" @@ -1348,6 +1341,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "crypto-common 0.1.7", + "subtle", ] [[package]] @@ -1364,13 +1358,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.7" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] @@ -1393,7 +1387,7 @@ checksum = "f88959de2d447fd3eddcf1909d1f19fe084e27a056a6904203dc5d8b9e771c1e" dependencies = [ "rust_decimal", "serde", - "thiserror 2.0.20", + "thiserror 2.0.18", "time", "winnow 0.6.26", ] @@ -1406,15 +1400,15 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" -version = "1.18.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "enumset" -version = "1.1.14" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccc5801fd11762e24d1e420d01d2ac518f2a2ca4329d4fbb6639f2412b6204e0" +checksum = "839c4174b41e75c8f7306110b2c51996a293b8d1d850edd529011841d9fede7d" dependencies = [ "enumset_derive", ] @@ -1428,7 +1422,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -1444,15 +1438,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] name = "event-listener" -version = "5.4.2" +version = "5.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" dependencies = [ + "concurrent-queue", "parking", "pin-project-lite", ] @@ -1469,11 +1464,10 @@ dependencies = [ [[package]] name = "eyre" -version = "0.6.14" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08309dbcc659c5549a24ddb9b27027640641b282ef5768267c7e675558986a3" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" dependencies = [ - "autocfg", "indenter", "once_cell", ] @@ -1486,7 +1480,7 @@ checksum = "c29b33a0187823f1fa88b36980227dc96c7504ede2288e7d2a77d9d6d88b260c" dependencies = [ "log", "once_cell", - "rand 0.9.5", + "rand 0.9.4", "tokio", ] @@ -1502,9 +1496,21 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.5.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "faststr" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca7d44d22004409a61c393afb3369c8f7bb74abcae49fe249ee01dcc3002113" +dependencies = [ + "bytes", + "rkyv", + "serde", + "simdutf8", +] [[package]] name = "figment" @@ -1524,9 +1530,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.12" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flatbuffers" @@ -1540,13 +1546,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.10" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", - "zlib-rs", ] [[package]] @@ -1572,18 +1577,18 @@ dependencies = [ [[package]] name = "foyer" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a59f42276891c0a4ce683fcda1aa1b4fd1065d68116c264e4bf49b9d318a0ed" +checksum = "3b0abc0b87814989efa711f9becd9f26969820e2d3905db27d10969c4bd45890" dependencies = [ "anyhow", - "asyncband", "equivalent", "foyer-common", "foyer-memory", "foyer-storage", "foyer-tokio", "futures-util", + "mea", "mixtrics", "pin-project", "serde", @@ -1592,9 +1597,9 @@ dependencies = [ [[package]] name = "foyer-common" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643b47d510032a01e5af70dca288b0c5ec531646c1a8392e793fb5b0c13bef30" +checksum = "a3db80d5dece93adb7ad709c84578794724a9cba342a7e566c3551c7ec626789" dependencies = [ "anyhow", "bincode", @@ -1619,21 +1624,21 @@ dependencies = [ [[package]] name = "foyer-memory" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae51f5089f3d9025ae77f17ea66d2489ac0f82fbb1d91462807bea174d486d82" +checksum = "db907f40a527ca2aa2f40a5f68b32ea58aa70f050cd233518e9ffd402cfba6ce" dependencies = [ "anyhow", - "asyncband", "bitflags", - "datasketches", + "cmsketch", "equivalent", "foyer-common", "foyer-intrusive-collections", "foyer-tokio", "futures-util", - "hashbrown 0.17.1", - "itertools 0.15.0", + "hashbrown 0.16.1", + "itertools 0.14.0", + "mea", "mixtrics", "parking_lot", "paste", @@ -1644,13 +1649,12 @@ dependencies = [ [[package]] name = "foyer-storage" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "118192f532ba013f0cdc607efc22324b9ed855baed185608af0daa986e835c6b" +checksum = "1983f1db3d0710e9c9d5fc116d9202dccd41a2d1e032572224f1aff5520aa958" dependencies = [ "allocator-api2", "anyhow", - "asyncband", "bytes", "core_affinity", "equivalent", @@ -1661,34 +1665,35 @@ dependencies = [ "fs4", "futures-core", "futures-util", - "hashbrown 0.17.1", + "hashbrown 0.16.1", "io-uring", - "itertools 0.15.0", + "itertools 0.14.0", "libc", "lz4", + "mea", "parking_lot", "pin-project", - "rand 0.10.2", + "rand 0.9.4", "serde", "tracing", "twox-hash", - "zstd 0.13.3", + "zstd", ] [[package]] name = "foyer-tokio" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6741d1133dfaab64d3f8a9290bbfed3685084add28f8b6ee7a6264aa4dc26918" +checksum = "f6577b05a7ffad0db555aedf00bfe52af818220fc4c1c3a7a12520896fc38627" dependencies = [ "tokio", ] [[package]] name = "fs-err" -version = "3.3.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91aa448ca50d7e79433bdf3ee8d99215430d2ec02ade5aefab2a073a1822e8a" +checksum = "73fde052dbfc920003cfd2c8e2c6e6d4cc7c1091538c3a24226cec0665ab08c0" dependencies = [ "autocfg", "tokio", @@ -1712,9 +1717,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" dependencies = [ "futures-channel", "futures-core", @@ -1727,9 +1732,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", "futures-sink", @@ -1737,15 +1742,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-executor" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" dependencies = [ "futures-core", "futures-task", @@ -1754,38 +1759,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] name = "futures-macro" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] name = "futures-sink" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" [[package]] name = "futures-task" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-util" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-channel", "futures-core", @@ -1818,7 +1823,7 @@ dependencies = [ "libc", "wasi", "wasm-bindgen", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -1828,9 +1833,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 5.3.0", "wasip2", + "wasm-bindgen", ] [[package]] @@ -1840,11 +1847,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", - "wasm-bindgen", ] [[package]] @@ -1854,7 +1859,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" dependencies = [ "polyval", - "zeroize", ] [[package]] @@ -1871,16 +1875,16 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.19" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.5.0", + "http 1.4.2", "indexmap", "slab", "tokio", @@ -1900,6 +1904,17 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -1925,9 +1940,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.5.3" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "hex" @@ -1945,6 +1960,15 @@ dependencies = [ "vsimd", ] +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "hmac" version = "0.13.0" @@ -1967,9 +1991,9 @@ dependencies = [ [[package]] name = "http" -version = "1.5.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -1988,24 +2012,24 @@ dependencies = [ [[package]] name = "http-body" -version = "1.1.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.5.0", + "http 1.4.2", ] [[package]] name = "http-body-util" -version = "0.1.5" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "pin-project-lite", ] @@ -2029,26 +2053,26 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" -version = "0.4.15" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" dependencies = [ "typenum", ] [[package]] name = "hyper" -version = "1.11.1" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", "h2", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -2064,7 +2088,7 @@ version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.5.0", + "http 1.4.2", "hyper", "hyper-util", "rustls", @@ -2080,12 +2104,12 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", "futures-util", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "hyper", "ipnet", "libc", @@ -2123,9 +2147,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", @@ -2137,9 +2161,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -2150,9 +2174,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -2164,17 +2188,16 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ - "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -2185,15 +2208,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.3.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -2239,9 +2262,9 @@ checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" [[package]] name = "indexmap" -version = "2.14.2" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -2264,9 +2287,9 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.15" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" +checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" dependencies = [ "bitflags", "cfg-if", @@ -2275,9 +2298,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.2" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "is_terminal_polyfill" @@ -2321,7 +2344,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.20", + "thiserror 2.0.18", "walkdir", "windows-link 0.2.1", ] @@ -2336,7 +2359,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn 2.0.119", + "syn", ] [[package]] @@ -2355,24 +2378,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "jobserver" -version = "0.1.35" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ - "getrandom 0.4.3", + "getrandom 0.3.4", "libc", ] [[package]] name = "js-sys" -version = "0.3.105" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" dependencies = [ "cfg-if", "futures-util", @@ -2387,9 +2410,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.189" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libloading" @@ -2409,22 +2432,22 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "linkme" -version = "0.3.37" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +checksum = "e83272d46373fb8decca684579ac3e7c8f3d71d4cc3aa693df8759e260ae41cf" dependencies = [ "linkme-impl", ] [[package]] name = "linkme-impl" -version = "0.3.37" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +checksum = "32d59e20403c7d08fe62b4376edfe5c7fb2ef1e6b1465379686d0f21c8df444b" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] @@ -2441,9 +2464,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" @@ -2456,15 +2479,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.34" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "lru" -version = "0.18.4" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" +checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" dependencies = [ "hashbrown 0.17.1", ] @@ -2510,7 +2533,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06138aebc1442e86b201f492616a51690a758d7b75253162f9e9208382ca2bb7" dependencies = [ "libc", - "rand 0.10.2", + "rand 0.10.1", "tokio", "turmoil", ] @@ -2541,10 +2564,19 @@ dependencies = [ ] [[package]] -name = "memchr" -version = "2.8.3" +name = "mea" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +checksum = "2640d335e7273dacdcf51044026139b2e269c3bb0dfc3f8cb3496b85e3f6a42c" +dependencies = [ + "slab", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "memoffset" @@ -2569,9 +2601,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.9.1" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", "simd-adler32", @@ -2579,9 +2611,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.3" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -2598,6 +2630,26 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "munge" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" +dependencies = [ + "munge_macro", +] + +[[package]] +name = "munge_macro" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "nix" version = "0.31.3" @@ -2644,14 +2696,14 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] name = "num-bigint" -version = "0.4.8" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", @@ -2665,9 +2717,9 @@ checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" -version = "0.1.47" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ "num-traits", ] @@ -2719,13 +2771,13 @@ dependencies = [ [[package]] name = "object_store" -version = "0.14.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50" +checksum = "765784b4390c6bcf80316e5a22f4e3661b639c9d8c83246856643c27d8ce9dbe" dependencies = [ "async-trait", "aws-lc-rs", - "base64 0.22.1", + "base64", "bytes", "chrono", "crc-fast", @@ -2733,7 +2785,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http 1.5.0", + "http 1.4.2", "http-body-util", "humantime", "hyper", @@ -2742,14 +2794,14 @@ dependencies = [ "nix", "parking_lot", "percent-encoding", - "quick-xml", - "rand 0.10.2", + "quick-xml 0.40.1", + "rand 0.10.1", "reqwest", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.20", + "thiserror 2.0.18", "tokio", "tracing", "url", @@ -2807,7 +2859,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -2871,16 +2923,16 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "pem" -version = "4.0.0" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64 0.23.1", + "base64", "serde_core", ] @@ -2907,7 +2959,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -2924,27 +2976,26 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "polyval" -version = "0.7.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" +checksum = "7dfc63250416fea14f5749b90725916a6c903f599d51cb635aa7a52bfd03eede" dependencies = [ "cpubits", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", "universal-hash", - "zeroize", ] [[package]] name = "potential_utf" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -2966,9 +3017,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.107" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] @@ -2981,7 +3032,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", "version_check", "yansi", ] @@ -3022,7 +3073,7 @@ dependencies = [ "parking_lot", "procfs", "protobuf", - "thiserror 2.0.20", + "thiserror 2.0.18", ] [[package]] @@ -3045,7 +3096,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -3068,6 +3119,36 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "ptr_meta" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quick-xml" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quick-xml" version = "0.41.0" @@ -3080,9 +3161,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.11" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" dependencies = [ "bytes", "cfg_aliases", @@ -3092,7 +3173,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.20", + "thiserror 2.0.18", "tokio", "tracing", "web-time", @@ -3100,22 +3181,21 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.17" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.4.3", + "getrandom 0.3.4", "lru-slab", - "rand 0.10.2", - "rand_pcg", + "rand 0.9.4", "ring", "rustc-hash", "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.20", + "thiserror 2.0.18", "tinyvec", "tracing", "web-time", @@ -3123,23 +3203,23 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.15" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" dependencies = [ "cfg_aliases", "libc", "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.47" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -3157,10 +3237,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] -name = "rand" -version = "0.8.8" +name = "rancor" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" +dependencies = [ + "ptr_meta", +] + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -3169,9 +3258,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.5" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -3179,9 +3268,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.2" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -3239,16 +3328,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.5", -] - -[[package]] -name = "rand_pcg" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" -dependencies = [ - "rand_core 0.10.1", + "rand 0.9.4", ] [[package]] @@ -3262,9 +3342,9 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.14.10" +version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055" dependencies = [ "aws-lc-rs", "pem", @@ -3285,41 +3365,29 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.27" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.27" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", -] - -[[package]] -name = "regex" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", + "syn", ] [[package]] name = "regex-automata" -version = "0.4.18" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -3339,18 +3407,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] -name = "reqwest" -version = "0.13.5" +name = "rend" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" + +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64 0.23.1", + "base64", "bytes", "futures-core", "futures-util", "h2", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-rustls", @@ -3392,10 +3466,39 @@ dependencies = [ ] [[package]] -name = "rust_decimal" -version = "1.43.0" +name = "rkyv" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7653272e75dcac41dc199fbea6f5797633994fafd339943c06c9af16bf29cd3a" +checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" +dependencies = [ + "bytes", + "hashbrown 0.17.1", + "indexmap", + "munge", + "ptr_meta", + "rancor", + "rend", + "rkyv_derive", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "rust_decimal" +version = "1.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be2a24f50780bc85f09cc6ac299bdf1424302742d77221106859c9d8b102126a" dependencies = [ "arrayvec", "num-traits", @@ -3403,9 +3506,9 @@ dependencies = [ [[package]] name = "rustc-hash" -version = "2.1.3" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "rustc_version" @@ -3458,14 +3561,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", "log", @@ -3490,9 +3593,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.1" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "web-time", "zeroize", @@ -3516,7 +3619,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3527,9 +3630,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", @@ -3539,9 +3642,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.23" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" @@ -3551,7 +3654,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s2-api" -version = "0.31.5" +version = "0.31.1" dependencies = [ "axum", "base64ct", @@ -3560,23 +3663,24 @@ dependencies = [ "flate2", "futures-core", "futures-util", - "http 1.5.0", + "http 1.4.2", "itertools 0.15.0", "mime", "prost", "s2-common", "serde", "serde_json", + "sonic-rs", "strum", - "thiserror 2.0.20", + "thiserror 2.0.18", "time", "tokio-util", - "zstd 0.14.0", + "zstd", ] [[package]] name = "s2-common" -version = "0.41.3" +version = "0.41.1" dependencies = [ "axum", "base64ct", @@ -3584,18 +3688,18 @@ dependencies = [ "clap", "compact_str", "enumset", - "http 1.5.0", - "rand 0.10.2", + "http 1.4.2", + "rand 0.10.1", "secrecy", "serde", "strum", - "thiserror 2.0.20", + "thiserror 2.0.18", "time", ] [[package]] name = "s2-lite" -version = "0.42.13" +version = "0.42.7" dependencies = [ "async-stream", "async-trait", @@ -3609,13 +3713,13 @@ dependencies = [ "dashmap", "eyre", "futures", - "http 1.5.0", + "http 1.4.2", "indexmap", "itertools 0.15.0", "parking_lot", "prometheus", "prost", - "rand 0.10.2", + "rand 0.10.1", "rcgen", "rustls", "s2-api", @@ -3626,12 +3730,12 @@ dependencies = [ "serde_json", "slatedb", "strum", - "thiserror 2.0.20", + "thiserror 2.0.18", "tikv-jemallocator", "time", "tokio", "tokio-util", - "tower-http 0.7.1", + "tower-http 0.7.0", "tracing", "tracing-subscriber", ] @@ -3640,7 +3744,6 @@ dependencies = [ name = "s2-resource-spec" version = "0.2.2" dependencies = [ - "compact_str", "humantime", "s2-common", "schemars", @@ -3650,25 +3753,24 @@ dependencies = [ [[package]] name = "s2-sdk" -version = "0.34.9" +version = "0.34.3" dependencies = [ "async-compression", "async-stream", "async-trait", "bytes", - "compact_str", "futures-core", "futures-util", "h2", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-rustls", "hyper-util", "pin-project-lite", "prost", - "rand 0.10.2", + "rand 0.10.1", "rustls", "s2-api", "s2-common", @@ -3676,7 +3778,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.20", + "thiserror 2.0.18", "time", "tokio", "tokio-muxt", @@ -3701,12 +3803,12 @@ dependencies = [ "fastrand", "futures", "getrandom 0.2.15", - "http 1.5.0", + "http 1.4.2", "http-body-util", "hyper", "hyper-util", "mad-turmoil", - "rand 0.10.2", + "rand 0.10.1", "s2-lite", "s2-sdk", "s2-verification", @@ -3722,17 +3824,17 @@ dependencies = [ [[package]] name = "s2-storage" -version = "0.2.5" +version = "0.2.4" dependencies = [ "aegis", "aes-gcm", "blake3", "bytes", - "rand 0.10.2", + "rand 0.10.1", "s2-common", "secrecy", "serde", - "thiserror 2.0.20", + "thiserror 2.0.18", ] [[package]] @@ -3744,7 +3846,7 @@ dependencies = [ "clap", "eyre", "futures", - "rand 0.8.8", + "rand 0.8.6", "s2-sdk", "serde", "serde_json", @@ -3757,9 +3859,9 @@ dependencies = [ [[package]] name = "s3s" -version = "0.15.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "652451a66fefda01a13dc7df24aa2af9e70c7058f98bc08fe2f7c552eaa9f1e3" +checksum = "abe1bd31748cb69848c2cf4028cecdadf5f8299ef3b02a83e21b20e7bda3aba9" dependencies = [ "arc-swap", "arrayvec", @@ -3773,9 +3875,9 @@ dependencies = [ "crc-fast", "futures", "hex-simd", - "hmac", - "http 1.5.0", - "http-body 1.1.0", + "hmac 0.13.0", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "httparse", "hyper", @@ -3786,18 +3888,17 @@ dependencies = [ "nom 8.0.0", "numeric_cast", "pin-project-lite", - "quick-xml", - "regex", + "quick-xml 0.41.0", "serde", "serde_json", "serde_urlencoded", "sha1 0.11.0", - "sha2", + "sha2 0.11.0", "smallvec", "std-next", "subtle", "sync_wrapper", - "thiserror 2.0.20", + "thiserror 2.0.18", "time", "tokio", "tower", @@ -3805,7 +3906,6 @@ dependencies = [ "transform-stream", "url", "urlencoding", - "xxhash-rust", "zeroize", ] @@ -3829,9 +3929,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.2" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ "dyn-clone", "ref-cast", @@ -3842,14 +3942,14 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "1.2.2" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" +checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 3.0.5", + "syn", ] [[package]] @@ -3904,9 +4004,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.229" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", "serde_derive", @@ -3914,40 +4014,40 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.229" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.229" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] name = "serde_derive_internals" -version = "0.30.0" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -4003,9 +4103,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.7" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", @@ -4019,10 +4119,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", "digest 0.11.3", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha2" version = "0.11.0" @@ -4030,7 +4141,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", "digest 0.11.3", ] @@ -4061,15 +4172,15 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.10" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] name = "simd_cesu8" -version = "1.2.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" dependencies = [ "rustc_version", "simdutf8", @@ -4095,9 +4206,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "slatedb" -version = "0.16.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb40332f41e231926df3a0ef742c05316b43368ba4b520433d2a1eba1ab00f0f" +checksum = "35ca56b01922b15aa69fe3abb62cadc985d86032c9647e4606e211c4da751a76" dependencies = [ "async-channel", "async-trait", @@ -4121,7 +4232,7 @@ dependencies = [ "object_store", "ouroboros", "parking_lot", - "rand 0.9.5", + "rand 0.9.4", "serde", "serde_json", "siphasher", @@ -4137,19 +4248,19 @@ dependencies = [ "url", "uuid", "walkdir", - "zstd 0.13.3", + "zstd", ] [[package]] name = "slatedb-common" -version = "0.16.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b77f238f348e95e1653a5235f880ac703011b82582bd556f2aba405abf8180e" +checksum = "0aa8de522ff46a0f9b5a66f45e650d125421d4d91990933591092f9d010c40d1" dependencies = [ "chrono", "log", "object_store", - "rand 0.9.5", + "rand 0.9.4", "rand_xoshiro", "serde", "thread_local", @@ -4158,9 +4269,9 @@ dependencies = [ [[package]] name = "slatedb-txn-obj" -version = "0.16.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad63b6c250219e26d71f497820f970274c301f896366daa56915f2a05df0e0a" +checksum = "d85fd9c0c86dd4954524fa8238d0548bd80f4a9a66983cbde3728402d339993e" dependencies = [ "async-trait", "bytes", @@ -4181,18 +4292,18 @@ checksum = "88414a5ca1f85d82cc34471e975f0f74f6aa54c40f062efa42c0080e7f763f81" [[package]] name = "smallvec" -version = "1.16.0" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" -version = "0.6.5" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4202,10 +4313,49 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45e14297decde697ddf377c25752aead0927d5cfc89c2684d2af96901a4ceeea" [[package]] -name = "spin" -version = "0.10.1" +name = "sonic-number" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" +checksum = "3775c3390edf958191f1ab1e8c5c188907feebd0f3ce1604cb621f72961dbf32" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "sonic-rs" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d971cc77a245ccf1756dbd1a87c3e7f709c0191464096510d43eec056d0f2c4f" +dependencies = [ + "ahash", + "bumpalo", + "bytes", + "cfg-if", + "faststr", + "itoa", + "ref-cast", + "serde", + "simdutf8", + "sonic-number", + "sonic-simd", + "thiserror 2.0.18", + "zmij", +] + +[[package]] +name = "sonic-simd" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f99e664ecd2d85a68c87e3c7a3cfe691f647ea9e835de984aba4d54a41f817d4" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" [[package]] name = "stable_deref_trait" @@ -4226,7 +4376,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04082e93ed1a06debd9148c928234b46d2cf260bc65f44e1d1d3fa594c5beebc" dependencies = [ "simdutf8", - "thiserror 2.0.20", + "thiserror 2.0.18", ] [[package]] @@ -4253,7 +4403,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4264,20 +4414,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.119" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "3.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -4301,7 +4440,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4329,11 +4468,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.20" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.20", + "thiserror-impl 2.0.18", ] [[package]] @@ -4344,25 +4483,25 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "thiserror-impl" -version = "2.0.20" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] name = "thread_local" -version = "1.1.10" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", ] @@ -4389,9 +4528,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.55" +version = "0.3.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" dependencies = [ "deranged", "num-conv", @@ -4409,9 +4548,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.32" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" dependencies = [ "num-conv", "time-core", @@ -4419,9 +4558,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.4" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -4429,9 +4568,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.13.2" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -4461,13 +4600,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.2" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] @@ -4482,9 +4621,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.5" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ "rustls", "tokio", @@ -4492,9 +4631,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.19" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" dependencies = [ "futures-core", "pin-project-lite", @@ -4503,16 +4642,15 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", "futures-sink", "futures-util", "hashbrown 0.15.5", - "libc", "pin-project-lite", "tokio", ] @@ -4583,8 +4721,8 @@ dependencies = [ "bitflags", "bytes", "futures-util", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "pin-project-lite", "tower", "tower-layer", @@ -4594,16 +4732,16 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.7.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c" +checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" dependencies = [ "async-compression", "bitflags", "bytes", "futures-core", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "percent-encoding", "pin-project-lite", @@ -4646,7 +4784,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4724,7 +4862,7 @@ checksum = "0061c28f1469e94771bb8aa626ce6b29265c2fb5034115046a170b0cba2e33d2" dependencies = [ "bytes", "indexmap", - "rand 0.9.5", + "rand 0.9.4", "rand_distr", "scoped-tls", "tokio", @@ -4733,9 +4871,12 @@ dependencies = [ [[package]] name = "twox-hash" -version = "2.1.4" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +dependencies = [ + "rand 0.9.4", +] [[package]] name = "typenum" @@ -4749,7 +4890,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" dependencies = [ - "rand 0.9.5", + "rand 0.9.4", "serde", "web-time", ] @@ -4829,13 +4970,13 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.26.1" +version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ "getrandom 0.4.3", "js-sys", - "rand 0.10.2", + "rand 0.10.1", "serde_core", "wasm-bindgen", ] @@ -4894,9 +5035,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.128" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" dependencies = [ "cfg-if", "once_cell", @@ -4907,9 +5048,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.78" +version = "0.4.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" dependencies = [ "js-sys", "wasm-bindgen", @@ -4917,9 +5058,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.128" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4927,22 +5068,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.128" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.5", + "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.128" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" dependencies = [ "unicode-ident", ] @@ -4962,9 +5103,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.105" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" dependencies = [ "js-sys", "wasm-bindgen", @@ -4982,9 +5123,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" dependencies = [ "rustls-pki-types", ] @@ -5011,7 +5152,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5087,7 +5228,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -5098,7 +5239,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -5165,7 +5306,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -5174,7 +5315,16 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", ] [[package]] @@ -5192,14 +5342,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link 0.2.1", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -5217,48 +5384,96 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "0.6.26" @@ -5285,9 +5500,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.4" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "x509-parser" @@ -5303,7 +5518,7 @@ dependencies = [ "nom 7.1.3", "oid-registry", "rusticata-macros", - "thiserror 2.0.20", + "thiserror 2.0.18", "time", ] @@ -5315,9 +5530,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" -version = "0.8.18" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" [[package]] name = "yansi" @@ -5354,28 +5569,28 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.57" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.57" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -5395,7 +5610,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", "synstructure", ] @@ -5407,9 +5622,9 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.5" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -5418,9 +5633,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.8" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -5429,26 +5644,20 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.6" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] -[[package]] -name = "zlib-rs" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" - [[package]] name = "zmij" -version = "1.0.23" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" [[package]] name = "zstd" @@ -5456,41 +5665,23 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe 7.3.0", -] - -[[package]] -name = "zstd" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e" -dependencies = [ - "zstd-safe 8.0.0", + "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.3.0" +version = "7.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-safe" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.1.0+zstd.1.5.7" +version = "2.0.16+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" dependencies = [ "cc", "pkg-config", diff --git a/sim/Cargo.toml b/sim/Cargo.toml index 1293bd1e..2b537aad 100644 --- a/sim/Cargo.toml +++ b/sim/Cargo.toml @@ -19,17 +19,17 @@ path = "src/main.rs" async-trait = "0.1" axum = "0.8" blake3 = "1.8" -bytes = "1.12" -bytesize = "2.7" +bytes = "1.11" +bytesize = "2.3" clap = { version = "4.6", features = ["derive", "env"] } eyre = "0.6" fastrand = "2" futures = "0.3" # Activates determinism on Linux in the patched getrandom (see [patch.crates-io]). getrandom = { version = "0.2", features = ["deterministic-simulation"] } -http = "1.5" +http = "1.4" http-body-util = "0.1" -hyper = { version = "1.11", features = ["client", "server", "http1", "http2"] } +hyper = { version = "1.9", features = ["client", "server", "http1", "http2"] } hyper-util = { version = "0.1", features = [ "client-legacy", "http1", @@ -46,10 +46,10 @@ s2-sdk = { path = "../sdk", features = ["_hidden"] } # The Go checker built in CI must be pinned to the same rev (see the # S2_VERIFICATION_REV env in .github/workflows/ci.yml). s2-verification = { git = "https://github.com/s2-streamstore/s2-verification", rev = "b4af8c8ef4965d9b335101c422eadb33f3169004" } -s3s = "0.15.0" +s3s = "0.14.1" serde_json = "1.0" -slatedb = { version = "0.16.0", features = ["lz4", "zstd"] } -tokio = { version = "1.53", features = ["macros", "rt", "sync", "time", "io-util"] } +slatedb = { version = "0.15.0", features = ["lz4", "zstd"] } +tokio = { version = "1.52", features = ["macros", "rt", "sync", "time", "io-util"] } tower = "0.5" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/storage/CHANGELOG.md b/storage/CHANGELOG.md index aeb91386..3719b3b7 100644 --- a/storage/CHANGELOG.md +++ b/storage/CHANGELOG.md @@ -2,22 +2,6 @@ All notable changes to this project will be documented in this file. -## [0.2.6] - 2026-09-25 - -### Miscellaneous Tasks - -- Updated the following local packages: s2-common - - - -## [0.2.5] - 2026-09-22 - -### Miscellaneous Tasks - -- Update Cargo.toml dependencies - - - ## [0.2.4] - 2026-07-22 ### Miscellaneous Tasks diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 1fccbb72..4bd31772 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-storage" -version = "0.2.6" +version = "0.2.4" description = "Storage-layer internals shared by S2 server implementations" edition.workspace = true license.workspace = true diff --git a/testcontainers/CHANGELOG.md b/testcontainers/CHANGELOG.md index 6d8b7d6d..aee84165 100644 --- a/testcontainers/CHANGELOG.md +++ b/testcontainers/CHANGELOG.md @@ -2,38 +2,6 @@ All notable changes to this project will be documented in this file. -## [0.43.0] - 2026-09-25 - - - -## [0.42.14] - 2026-09-24 - - - -## [0.42.13] - 2026-09-22 - -### Miscellaneous Tasks - -- Update Cargo.toml dependencies - - - -## [0.42.12] - 2026-09-16 - - - -## [0.42.11] - 2026-09-11 - - - -## [0.42.10] - 2026-09-11 - - - -## [0.42.9] - 2026-09-10 - - - ## [0.42.8] - 2026-09-01 diff --git a/testcontainers/Cargo.toml b/testcontainers/Cargo.toml index 23647de1..3dfb58e1 100644 --- a/testcontainers/Cargo.toml +++ b/testcontainers/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-testcontainers" -version = "0.43.0" +version = "0.42.8" description = "Testcontainers helpers for the S2 Docker image" edition.workspace = true license.workspace = true