diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index f27aa58b..00000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,5 +0,0 @@ -[unstable] -min-publish-age = true - -[registry] -global-min-publish-age = "7 days" diff --git a/.github/actions/rust-dependency-cooldown/.gitignore b/.github/actions/rust-dependency-cooldown/.gitignore deleted file mode 100644 index c18dd8d8..00000000 --- a/.github/actions/rust-dependency-cooldown/.gitignore +++ /dev/null @@ -1 +0,0 @@ -__pycache__/ diff --git a/.github/actions/rust-dependency-cooldown/README.md b/.github/actions/rust-dependency-cooldown/README.md deleted file mode 100644 index c36831f3..00000000 --- a/.github/actions/rust-dependency-cooldown/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Rust dependency cooldown - -This action checks newly locked crates.io versions against the caller's -`registry.global-min-publish-age` setting. S2-owned crates listed in -`first-party-crates.txt` are exempt from the publication-age check. - -## Security exceptions - -`security-exceptions.toml` records reviewed exceptions for individual security -releases. Each entry requires an exact crate name and version, an advisory -identifier or URL, and a reason: - -```toml -[[exception]] -crate = "rustls" -version = "0.23.45" -advisory = "RUSTSEC-2026-0285" -reason = "Fixes TLS 1.3 handshake messages accepted at the wrong encryption level" -``` - -Review the advisory and confirm that the exact release fixes it before adding an -entry. Ranges, wildcards, duplicate entries, and malformed configuration are -rejected. The action prints the crate, version, advisory, and reason whenever it -uses an exception. The exception waives only publication age; other dependency -checks, including `cargo deny`, continue to apply. - -The exception file is distributed with this action. After merging an exception, -update consuming repositories' pinned action or reusable-workflow commit to pick -it up. Future releases of the same crate still have to satisfy the cooldown. - -Entries need no expiry field: an approved release follows the normal age check -once it is old enough. Old entries may be removed in a later cleanup. An empty -file or `exception = []` means there are no security exceptions. - -Cargo also enforces publication age during dependency resolution. To select an -approved release, override that resolver check for the targeted update command: - -```sh -CARGO_RESOLVER_INCOMPATIBLE_PUBLISH_AGE=allow cargo +nightly update -p rustls --precise 0.23.45 -``` - -Run the cooldown action on the resulting lockfile to check every newly selected -version against the exception list and normal age requirement. - -## Tests - -Run from the repository root: - -```sh -python3 -m unittest discover -s .github/actions/rust-dependency-cooldown -p 'test_*.py' -v -``` diff --git a/.github/actions/rust-dependency-cooldown/action.yml b/.github/actions/rust-dependency-cooldown/action.yml deleted file mode 100644 index 72093d91..00000000 --- a/.github/actions/rust-dependency-cooldown/action.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Rust dependency cooldown gate -description: Reject new crates.io versions that are inside the publication cooldown - -inputs: - base-sha: - description: Pull request base commit - required: true - -runs: - using: composite - steps: - - name: Check dependency publish age - shell: bash - env: - ACTION_PATH: ${{ github.action_path }} - BASE_SHA: ${{ inputs.base-sha }} - run: | - env -i \ - HOME="$RUNNER_TEMP" \ - PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \ - python3 "$ACTION_PATH/check.py" \ - --repo-root "$GITHUB_WORKSPACE" \ - --config "$GITHUB_WORKSPACE/.cargo/config.toml" \ - --allowlist "$ACTION_PATH/first-party-crates.txt" \ - "$BASE_SHA" diff --git a/.github/actions/rust-dependency-cooldown/check.py b/.github/actions/rust-dependency-cooldown/check.py deleted file mode 100644 index 53f7d0b0..00000000 --- a/.github/actions/rust-dependency-cooldown/check.py +++ /dev/null @@ -1,292 +0,0 @@ -#!/usr/bin/env python3 -"""Reject newly locked crates.io versions that are too new.""" - -from __future__ import annotations - -import argparse -import json -import re -import subprocess -import sys -import time -import tomllib -import urllib.error -import urllib.request -from datetime import datetime, timedelta, timezone -from pathlib import Path - - -ACTION_ROOT = Path(__file__).resolve().parent -REPO_ROOT = Path.cwd() -CONFIG = REPO_ROOT / ".cargo" / "config.toml" -ALLOWLIST = ACTION_ROOT / "first-party-crates.txt" -SECURITY_EXCEPTIONS = ACTION_ROOT / "security-exceptions.toml" -CRATES_IO_SOURCE = "registry+https://github.com/rust-lang/crates.io-index" -INDEX_BASE = "https://index.crates.io" -USER_AGENT = "s2 minimum-publish-age check (github.com/s2-streamstore/s2)" -RETRY_ATTEMPTS = 4 -RETRY_BASE_DELAY_SECONDS = 2 -UNIT_SECONDS = { - "second": 1, - "seconds": 1, - "minute": 60, - "minutes": 60, - "hour": 3600, - "hours": 3600, - "day": 86400, - "days": 86400, - "week": 604800, - "weeks": 604800, - "month": 2592000, - "months": 2592000, -} -RELEVANT_PATHS = ( - ":(glob)**/Cargo.lock", - ".cargo/config.toml", - ".github/actions/rust-dependency-cooldown", - ".github/workflows/rust-dependency-cooldown.yml", -) - - -def minimum_age() -> tuple[timedelta, str]: - with CONFIG.open("rb") as config_file: - raw = tomllib.load(config_file).get("registry", {}).get("global-min-publish-age") - if not isinstance(raw, str): - raise ValueError(f"registry.global-min-publish-age is not set in {CONFIG}") - value = raw.strip() - if value == "0": - return timedelta(0), value - match = re.fullmatch(r"(\d+)\s+(\w+)", value) - if match is None or match.group(2) not in UNIT_SECONDS: - raise ValueError(f"cannot parse global-min-publish-age = {value!r}") - return timedelta(seconds=int(match.group(1)) * UNIT_SECONDS[match.group(2)]), value - - -def allowed_crates() -> set[str]: - return { - name - for line in ALLOWLIST.read_text().splitlines() - if (name := line.split("#", 1)[0].strip()) - } - - -def security_exceptions() -> dict[tuple[str, str], dict[str, str]]: - with SECURITY_EXCEPTIONS.open("rb") as exceptions_file: - document = tomllib.load(exceptions_file) - if document.keys() - {"exception"}: - raise ValueError(f"unexpected fields in {SECURITY_EXCEPTIONS}") - entries = document.get("exception", []) - if not isinstance(entries, list): - raise ValueError(f"expected [[exception]] entries in {SECURITY_EXCEPTIONS}") - required_fields = {"crate", "version", "advisory", "reason"} - exceptions: dict[tuple[str, str], dict[str, str]] = {} - for index, entry in enumerate(entries, start=1): - if not isinstance(entry, dict) or entry.keys() != required_fields: - raise ValueError(f"security exception {index} must contain {sorted(required_fields)}") - for field, value in entry.items(): - if not isinstance(value, str) or not value or value != value.strip(): - raise ValueError(f"security exception {index} has invalid {field}") - if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]*", entry["crate"]) is None: - raise ValueError(f"security exception {index} requires an exact crate name") - if re.fullmatch( - r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?", - entry["version"], - ) is None: - raise ValueError(f"security exception {index} requires an exact version") - key = (entry["crate"], entry["version"]) - if key in exceptions: - raise ValueError(f"duplicate security exception for {key[0]} {key[1]}") - exceptions[key] = entry - return exceptions - - -def crates_io_versions(lock_text: str) -> set[tuple[str, str]]: - packages = tomllib.loads(lock_text).get("package", []) - return { - (package["name"], package["version"]) - for package in packages - if package.get("source") == CRATES_IO_SOURCE - } - - -def run_git(*args: str) -> subprocess.CompletedProcess[str]: - return subprocess.run( - ["git", *args], - cwd=REPO_ROOT, - capture_output=True, - text=True, - check=False, - ) - - -def lock_at(revision: str, path: str) -> str | None: - result = run_git("show", f"{revision}:{path}") - return result.stdout if result.returncode == 0 else None - - -def has_relevant_changes(base_ref: str) -> bool: - result = run_git("diff", "--quiet", base_ref, "HEAD", "--", *RELEVANT_PATHS) - if result.returncode == 1: - return True - if result.returncode != 0: - raise RuntimeError( - f"cannot compare committed Rust dependency cooldown paths: {result.stderr.strip()}" - ) - - result = run_git("diff", "--quiet", "HEAD", "--", *RELEVANT_PATHS) - if result.returncode == 1: - return True - if result.returncode != 0: - raise RuntimeError( - f"cannot compare working Rust dependency cooldown paths: {result.stderr.strip()}" - ) - - result = run_git("ls-files", "--others", "--exclude-standard", "--", *RELEVANT_PATHS) - if result.returncode: - raise RuntimeError( - f"cannot list untracked Rust dependency cooldown paths: {result.stderr.strip()}" - ) - return bool(result.stdout.strip()) - - -def index_url(name: str) -> str: - normalized = name.lower() - if len(normalized) == 1: - path = f"1/{normalized}" - elif len(normalized) == 2: - path = f"2/{normalized}" - elif len(normalized) == 3: - path = f"3/{normalized[0]}/{normalized}" - else: - path = f"{normalized[:2]}/{normalized[2:4]}/{normalized}" - return f"{INDEX_BASE}/{path}" - - -def fetch_index(name: str) -> str: - request = urllib.request.Request(index_url(name), headers={"User-Agent": USER_AGENT}) - last_error: Exception | None = None - for attempt in range(1, RETRY_ATTEMPTS + 1): - if attempt > 1: - time.sleep(RETRY_BASE_DELAY_SECONDS * (attempt - 1)) - try: - with urllib.request.urlopen(request, timeout=30) as response: - raw = response.read() - except urllib.error.HTTPError as error: - if error.code < 500 and error.code != 429: - raise RuntimeError(f"cannot read the crates.io index for {name}: {error}") from error - last_error = error - continue - except (urllib.error.URLError, TimeoutError) as error: - last_error = error - continue - try: - return raw.decode() - except UnicodeDecodeError as error: - raise RuntimeError(f"cannot read the crates.io index for {name}: {error}") from error - raise RuntimeError( - f"cannot read the crates.io index for {name} after {RETRY_ATTEMPTS} attempts: {last_error}" - ) from last_error - - -def publication_times(name: str) -> dict[str, datetime]: - times: dict[str, datetime] = {} - for line in fetch_index(name).splitlines(): - if not line.strip(): - continue - entry = json.loads(line) - if pubtime := entry.get("pubtime"): - times[entry["vers"]] = datetime.fromisoformat(pubtime.replace("Z", "+00:00")) - return times - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--repo-root", - type=Path, - default=Path.cwd(), - help="repository whose lock files are checked", - ) - parser.add_argument("--config", type=Path, help="Cargo configuration to read") - parser.add_argument("--allowlist", type=Path, help="exact first-party crate names") - parser.add_argument( - "base_ref", - nargs="?", - help="check only crates.io versions not present at this Git ref", - ) - return parser.parse_args() - - -def main() -> int: - global ALLOWLIST, CONFIG, REPO_ROOT - - args = parse_args() - REPO_ROOT = args.repo_root.resolve() - CONFIG = (args.config or REPO_ROOT / ".cargo" / "config.toml").resolve() - ALLOWLIST = (args.allowlist or ACTION_ROOT / "first-party-crates.txt").resolve() - try: - if args.base_ref and run_git("rev-parse", "--verify", "--quiet", args.base_ref).returncode: - raise ValueError(f"base ref {args.base_ref!r} is not available") - approved_exceptions = security_exceptions() - if args.base_ref and not has_relevant_changes(args.base_ref): - print("No Rust dependency cooldown files changed; check skipped.") - return 0 - - age_limit, age_text = minimum_age() - allowlist = allowed_crates() - - lockfile_result = run_git( - "ls-files", "--cached", "--others", "--exclude-standard", "*Cargo.lock" - ) - if lockfile_result.returncode: - raise RuntimeError(f"cannot list Cargo.lock files: {lockfile_result.stderr.strip()}") - lockfiles = lockfile_result.stdout.splitlines() - now = datetime.now(timezone.utc) - violations: list[str] = [] - checked = 0 - index_cache: dict[str, dict[str, datetime]] = {} - - for lockfile in sorted(lockfiles): - proposed = crates_io_versions((REPO_ROOT / lockfile).read_text()) - baseline: set[tuple[str, str]] = set() - if args.base_ref and (text := lock_at(args.base_ref, lockfile)) is not None: - baseline = crates_io_versions(text) - - for name, version in sorted(proposed - baseline): - if name in allowlist: - continue - if name not in index_cache: - index_cache[name] = publication_times(name) - pubtime = index_cache[name].get(version) - if pubtime is None: - raise RuntimeError(f"no publication time for {name} {version}") - checked += 1 - crate_age = now - pubtime - if crate_age < age_limit: - if exception := approved_exceptions.get((name, version)): - print( - f"Publication cooldown waived for {name} {version} ({lockfile}): " - f"{exception['advisory']} — {exception['reason']}" - ) - continue - violations.append( - f"{name} {version} ({lockfile}): published " - f"{crate_age.total_seconds() / 86400:.1f} days ago; " - f"minimum is {age_text}" - ) - except (OSError, ValueError, RuntimeError, json.JSONDecodeError, tomllib.TOMLDecodeError) as error: - print(f"minimum-publish-age error: {error}", file=sys.stderr) - return 2 - - if violations: - print("New crates.io versions are inside the publication cooldown:", file=sys.stderr) - for violation in violations: - print(f" - {violation}", file=sys.stderr) - return 1 - - print(f"Checked {checked} new crates.io version(s); publication cooldown policy satisfied.") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/actions/rust-dependency-cooldown/first-party-crates.txt b/.github/actions/rust-dependency-cooldown/first-party-crates.txt deleted file mode 100644 index f624704e..00000000 --- a/.github/actions/rust-dependency-cooldown/first-party-crates.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Crates that bypass only the minimum publish age. -# Use exact crate names. Only add crates that S2 publishes and controls. -s2-api -s2-cli -s2-common -s2-lite -s2-resource-spec -s2-sdk -s2-storage -s2-testcontainers diff --git a/.github/actions/rust-dependency-cooldown/security-exceptions.toml b/.github/actions/rust-dependency-cooldown/security-exceptions.toml deleted file mode 100644 index c0e3a044..00000000 --- a/.github/actions/rust-dependency-cooldown/security-exceptions.toml +++ /dev/null @@ -1,5 +0,0 @@ -[[exception]] -crate = "rustls" -version = "0.23.45" -advisory = "RUSTSEC-2026-0285" -reason = "Fixes TLS 1.3 handshake messages accepted at the wrong encryption level" diff --git a/.github/actions/rust-dependency-cooldown/test_check.py b/.github/actions/rust-dependency-cooldown/test_check.py deleted file mode 100644 index 313e59a5..00000000 --- a/.github/actions/rust-dependency-cooldown/test_check.py +++ /dev/null @@ -1,290 +0,0 @@ -from __future__ import annotations - -import importlib.util -import io -import json -import subprocess -import sys -import tempfile -import unittest -from contextlib import ExitStack, redirect_stderr, redirect_stdout -from datetime import datetime, timedelta, timezone -from pathlib import Path -from unittest.mock import patch - - -SPEC = importlib.util.spec_from_file_location( - "cooldown_check", Path(__file__).with_name("check.py") -) -assert SPEC is not None and SPEC.loader is not None -check = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(check) - -NOW = datetime(2026, 9, 16, tzinfo=timezone.utc) -APPROVED_EXCEPTION = { - "crate": "rustls", - "version": "0.23.45", - "advisory": "https://rustsec.org/advisories/RUSTSEC-2026-0285.html", - "reason": "First release fixing TLS 1.3 handshake encryption-level validation.", -} - - -def exception_document(*entries: dict[str, object]) -> str: - return "\n".join( - "[[exception]]\n" - + "\n".join(f"{name} = {json.dumps(value)}" for name, value in entry.items()) - + "\n" - for entry in entries - ) - - -class CooldownSecurityExceptionTests(unittest.TestCase): - def setUp(self) -> None: - temporary_directory = tempfile.TemporaryDirectory() - self.addCleanup(temporary_directory.cleanup) - temporary_root = Path(temporary_directory.name) - self.repo = temporary_root / "consumer" - self.repo.mkdir() - self.action = temporary_root / "trusted-action" - self.action.mkdir() - self.exceptions = self.action / "security-exceptions.toml" - self.exceptions.write_text(exception_document(APPROVED_EXCEPTION)) - (self.action / "first-party-crates.txt").write_text("s2-api\n") - (self.repo / ".cargo").mkdir() - (self.repo / ".cargo" / "config.toml").write_text( - '[registry]\nglobal-min-publish-age = "7 days"\n' - ) - self.write_lock() - self.git("init", "--quiet") - self.commit() - - def git(self, *arguments: str) -> None: - subprocess.run( - [ - "git", - "-c", - "user.name=Cooldown Tests", - "-c", - "user.email=cooldown-tests@example.invalid", - "-c", - "commit.gpgsign=false", - "-c", - "core.hooksPath=/dev/null", - *arguments, - ], - cwd=self.repo, - check=True, - capture_output=True, - text=True, - ) - - def commit(self) -> None: - self.git("add", ".") - self.git("commit", "--quiet", "-m", "test: record baseline") - - def write_lock(self, *packages: tuple[str, str, str]) -> None: - contents = "version = 3\n" - for name, version, source in packages: - contents += ( - "\n[[package]]\n" - f"name = {json.dumps(name)}\n" - f"version = {json.dumps(version)}\n" - f"source = {json.dumps(source)}\n" - ) - (self.repo / "Cargo.lock").write_text(contents) - - def run_gate( - self, - publication_times: dict[str, dict[str, datetime]] | None = None, - ) -> tuple[int, str, str, list[str]]: - published = publication_times or { - "rustls": {"0.23.45": NOW - timedelta(days=1)}, - } - output, errors = io.StringIO(), io.StringIO() - with ExitStack() as stack: - for name, value in { - "ACTION_ROOT": self.action, - "REPO_ROOT": self.repo, - "CONFIG": self.repo / ".cargo" / "config.toml", - "ALLOWLIST": self.action / "first-party-crates.txt", - "SECURITY_EXCEPTIONS": self.exceptions, - }.items(): - stack.enter_context(patch.object(check, name, value)) - stack.enter_context( - patch.object(sys, "argv", ["check.py", "--repo-root", str(self.repo), "HEAD"]) - ) - clock = stack.enter_context(patch.object(check, "datetime", wraps=datetime)) - clock.now.return_value = NOW - lookup = stack.enter_context( - patch.object(check, "publication_times", side_effect=published.__getitem__) - ) - stack.enter_context(redirect_stdout(output)) - stack.enter_context(redirect_stderr(errors)) - status = check.main() - lookups = [call.args[0] for call in lookup.call_args_list] - return status, output.getvalue(), errors.getvalue(), lookups - - def test_exact_security_exception_passes_and_reports_justification(self) -> None: - self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) - status, output, errors, _ = self.run_gate() - self.assertEqual(status, 0, errors) - for field in APPROVED_EXCEPTION.values(): - self.assertIn(field, output) - - def test_exception_does_not_cover_another_version_or_crate(self) -> None: - for name, version in [("rustls", "0.23.46"), ("another-crate", "0.23.45")]: - with self.subTest(crate=name, version=version): - self.write_lock((name, version, check.CRATES_IO_SOURCE)) - status, _, errors, _ = self.run_gate( - {name: {version: NOW - timedelta(days=1)}} - ) - self.assertEqual(status, 1) - self.assertIn(f"{name} {version}", errors) - - def test_exception_does_not_hide_an_unrelated_young_dependency(self) -> None: - self.write_lock( - ("rustls", "0.23.45", check.CRATES_IO_SOURCE), - ("another-crate", "1.0.0", check.CRATES_IO_SOURCE), - ) - status, output, errors, _ = self.run_gate( - { - "rustls": {"0.23.45": NOW - timedelta(days=1)}, - "another-crate": {"1.0.0": NOW - timedelta(days=1)}, - } - ) - self.assertEqual(status, 1) - self.assertIn(APPROVED_EXCEPTION["advisory"], output) - self.assertIn("another-crate 1.0.0", errors) - self.assertNotIn("rustls 0.23.45", errors) - - def test_consumer_repository_cannot_supply_its_own_exception(self) -> None: - unapproved = {**APPROVED_EXCEPTION, "version": "0.23.46"} - consumer_action = self.repo / ".github" / "actions" / "rust-dependency-cooldown" - consumer_action.mkdir(parents=True) - for directory in [self.repo, self.repo / ".cargo", consumer_action]: - (directory / "security-exceptions.toml").write_text(exception_document(unapproved)) - self.write_lock(("rustls", "0.23.46", check.CRATES_IO_SOURCE)) - status, _, errors, _ = self.run_gate( - {"rustls": {"0.23.46": NOW - timedelta(days=1)}} - ) - self.assertEqual(status, 1) - self.assertIn("rustls 0.23.46", errors) - - def test_malformed_security_exceptions_fail_closed(self) -> None: - documents = { - "invalid TOML": "[[exception]", - "not an array": "exception = 1\n", - "not a table": "exception = [1]\n", - "unknown top-level field": "exceptions = []\n", - } - for field in APPROVED_EXCEPTION: - missing = {name: value for name, value in APPROVED_EXCEPTION.items() if name != field} - documents[f"missing {field}"] = exception_document(missing) - for value in ["", " ", 123, f" {APPROVED_EXCEPTION[field]}"]: - documents[f"invalid {field}: {value!r}"] = exception_document( - {**APPROVED_EXCEPTION, field: value} - ) - documents["unknown field"] = exception_document({**APPROVED_EXCEPTION, "extra": "value"}) - documents["duplicate pair"] = exception_document(APPROVED_EXCEPTION, APPROVED_EXCEPTION) - self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) - for description, document in documents.items(): - with self.subTest(description=description): - self.exceptions.write_text(document) - status, _, errors, _ = self.run_gate() - self.assertEqual(status, 2) - self.assertIn("minimum-publish-age error", errors) - - def test_missing_action_exception_file_fails_closed(self) -> None: - self.exceptions.unlink() - self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) - status, _, errors, _ = self.run_gate() - self.assertEqual(status, 2) - self.assertIn("minimum-publish-age error", errors) - - def test_malformed_exceptions_fail_even_when_no_lockfile_changed(self) -> None: - self.exceptions.write_text("[[exception]") - status, _, errors, _ = self.run_gate() - self.assertEqual(status, 2) - self.assertIn("minimum-publish-age error", errors) - - def test_empty_exception_lists_do_not_exempt_young_dependencies(self) -> None: - self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) - for document in ["", "exception = []\n"]: - with self.subTest(document=document): - self.exceptions.write_text(document) - status, _, errors, _ = self.run_gate() - self.assertEqual(status, 1) - self.assertIn("rustls 0.23.45", errors) - - def test_exception_ranges_and_wildcards_are_rejected(self) -> None: - self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) - for field, value in [ - ("crate", "rustls*"), - ("version", "*"), - ("version", "0.23.*"), - ("version", ">=0.23.45"), - ]: - with self.subTest(field=field, value=value): - self.exceptions.write_text( - exception_document({**APPROVED_EXCEPTION, field: value}) - ) - status, _, errors, _ = self.run_gate() - self.assertEqual(status, 2) - self.assertIn("minimum-publish-age error", errors) - - def test_approved_version_still_requires_a_publication_time(self) -> None: - self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) - status, _, errors, _ = self.run_gate({"rustls": {}}) - self.assertEqual(status, 2) - self.assertIn("no publication time for rustls 0.23.45", errors) - - def test_mature_approved_version_uses_normal_age_check(self) -> None: - self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) - status, output, errors, lookups = self.run_gate( - {"rustls": {"0.23.45": NOW - timedelta(days=7)}} - ) - self.assertEqual(status, 0, errors) - self.assertNotIn(APPROVED_EXCEPTION["advisory"], output) - self.assertEqual(lookups, ["rustls"]) - - def test_age_boundary_for_ordinary_dependencies_is_unchanged(self) -> None: - self.write_lock(("another-crate", "1.0.0", check.CRATES_IO_SOURCE)) - for age, expected_status in [(timedelta(days=7), 0), (timedelta(days=7, seconds=-1), 1)]: - with self.subTest(age=age): - status, _, errors, _ = self.run_gate({"another-crate": {"1.0.0": NOW - age}}) - self.assertEqual(status, expected_status, errors) - - def test_first_party_allowlist_still_exempts_new_versions(self) -> None: - self.write_lock(("s2-api", "99.0.0", check.CRATES_IO_SOURCE)) - status, _, errors, lookups = self.run_gate() - self.assertEqual(status, 0, errors) - self.assertEqual(lookups, []) - - def test_other_sources_are_not_processed_as_security_exceptions(self) -> None: - for source in [ - "registry+https://example.invalid/index", - "git+https://example.invalid/rustls", - ]: - with self.subTest(source=source): - self.write_lock(("rustls", "0.23.45", source)) - status, output, errors, lookups = self.run_gate() - self.assertEqual(status, 0, errors) - self.assertNotIn(APPROVED_EXCEPTION["advisory"], output) - self.assertEqual(lookups, []) - - def test_existing_locked_versions_are_not_rechecked(self) -> None: - self.write_lock(("already-locked", "1.0.0", check.CRATES_IO_SOURCE)) - self.commit() - self.write_lock( - ("already-locked", "1.0.0", check.CRATES_IO_SOURCE), - ("another-crate", "1.0.0", check.CRATES_IO_SOURCE), - ) - status, _, errors, lookups = self.run_gate( - {"another-crate": {"1.0.0": NOW - timedelta(days=8)}} - ) - self.assertEqual(status, 0, errors) - self.assertEqual(lookups, ["another-crate"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/.github/workflows/build-s2-lite.yml b/.github/workflows/build-s2-lite.yml index aedcb617..bc294910 100644 --- a/.github/workflows/build-s2-lite.yml +++ b/.github/workflows/build-s2-lite.yml @@ -53,7 +53,7 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Build s2-lite - run: cargo build --locked --profile ci -p s2-lite + run: cargo build --profile ci -p s2-lite - name: Upload binary uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2da7b71d..4c12a723 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,12 +10,6 @@ env: CARGO_TERM_COLOR: always jobs: - rust-dependency-cooldown: - name: Rust dependency cooldown gate - permissions: - contents: read - uses: $/.github/workflows/rust-dependency-cooldown.yml - changes: name: Detect Changes runs-on: ubuntu-latest @@ -38,7 +32,7 @@ jobs: cli-schema-drift: name: CLI Schema Drift - needs: [changes, rust-dependency-cooldown] + needs: [changes] if: needs.changes.outputs.apply_schema == 'true' runs-on: ubuntu-latest steps: @@ -46,7 +40,7 @@ jobs: - uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Generate apply schema - run: cargo run --locked -q -p s2-cli -- apply --schema > /tmp/apply.schema.json + run: cargo run -q -p s2-cli -- apply --schema > /tmp/apply.schema.json - name: Check for schema drift run: diff -u cli/schema.json /tmp/apply.schema.json @@ -70,7 +64,9 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable - name: Verify Cargo.lock is up-to-date - run: cargo metadata --locked --format-version 1 >/dev/null + run: | + cargo metadata --locked --format-version 1 >/dev/null + cargo metadata --locked --manifest-path sim/Cargo.toml --format-version 1 >/dev/null sort: name: Cargo Sort @@ -92,7 +88,6 @@ jobs: clippy: name: Clippy - needs: rust-dependency-cooldown runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -106,15 +101,14 @@ jobs: uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - run: cargo clippy --locked --workspace --all-features --all-targets -- -D warnings --allow deprecated + - run: cargo clippy --workspace --all-features --all-targets -- -D warnings --allow deprecated - name: Clippy (simulator) env: RUSTFLAGS: --cfg tokio_unstable - run: cargo clippy --manifest-path sim/Cargo.toml --all-targets -- -D warnings --allow deprecated + run: cargo clippy --locked --manifest-path sim/Cargo.toml --all-targets -- -D warnings --allow deprecated test: name: Tests - needs: rust-dependency-cooldown runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -127,11 +121,10 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} - uses: taiki-e/install-action@d5f9268ff7620505a81ada10ddf18cdd72240185 # nextest - - run: cargo nextest run --locked --workspace --all-features --exclude s2-sdk --exclude s2-testcontainers -E 'not (package(s2-cli) & binary(integration))' + - run: cargo nextest run --workspace --all-features --exclude s2-sdk --exclude s2-testcontainers -E 'not (package(s2-cli) & binary(integration))' testcontainers: name: Testcontainers - needs: rust-dependency-cooldown runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -165,11 +158,10 @@ jobs: run: | docker run --rm ghcr.io/s2-streamstore/s2:${{ steps.image-version.outputs.value }} --version \ | grep -F "rev $S2_GIT_REV" - - run: cargo test --locked -p s2-testcontainers + - run: cargo test -p s2-testcontainers simulation: name: Simulation Tests - needs: rust-dependency-cooldown # arm64, deliberately: on x86_64 Linux, fastant (via slatedb -> foyer) runs # a pre-main TSC calibration loop that spins forever under mad-turmoil's # interposed clock_gettime, hanging the simulator at startup. On aarch64 @@ -194,7 +186,7 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Build simulator - run: cargo build --manifest-path sim/Cargo.toml --profile sim + run: cargo build --locked --manifest-path sim/Cargo.toml --profile sim # Determinism first: a broken meta test means seeds are not reproducible # and any linearizability failure would not be debuggable. Children @@ -310,21 +302,8 @@ jobs: exit 1 fi - - name: Test with persistent volume and WAL storage - run: | - helm template test-release charts/s2-lite-helm \ - --set persistentVolume.enabled=true \ - --set walStorage.persistentVolume.enabled=true \ - --dry-run > /dev/null - helm template test-release charts/s2-lite-helm \ - --set objectStorage.enabled=true \ - --set objectStorage.bucket=test-bucket \ - --set walStorage.bucket=test-wal-bucket \ - --dry-run > /dev/null - build-server: name: Build s2-lite - needs: rust-dependency-cooldown uses: ./.github/workflows/build-s2-lite.yml with: ref: ${{ github.sha }} @@ -369,7 +348,7 @@ jobs: "repo": "${{ github.repository }}", "ref": "${{ github.sha }}", "lang": "rust", - "test_cmd": "cargo test --locked -p s2-sdk --all-features -- --skip access_token --skip metrics" + "test_cmd": "cargo test -p s2-sdk --all-features -- --skip access_token --skip metrics" } ] @@ -383,11 +362,11 @@ jobs: - uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - name: Check docs - run: cargo doc --locked -p s2-sdk --all-features --no-deps + run: cargo doc -p s2-sdk --all-features --no-deps env: RUSTDOCFLAGS: "-D warnings" - name: Run tests - run: cargo test --locked -p s2-sdk --all-features + run: cargo test -p s2-sdk --all-features env: S2_ACCESS_TOKEN: ${{ secrets.S2_ACCESS_TOKEN_FOR_RUST_SDK_TESTS }} @@ -407,6 +386,6 @@ jobs: "repo": "${{ github.repository }}", "ref": "${{ github.sha }}", "lang": "rust", - "test_cmd": "cargo test --locked -p s2-cli --test integration -j 1" + "test_cmd": "cargo test -p s2-cli --test integration -j 1" } ] diff --git a/.github/workflows/release-cli.yml b/.github/workflows/release-cli.yml index 42fddca5..cf06b9c7 100644 --- a/.github/workflows/release-cli.yml +++ b/.github/workflows/release-cli.yml @@ -138,7 +138,7 @@ jobs: # an official release artifact for install-channel detection. S2_BUILD_CHANNEL: release S2_GIT_REV: ${{ steps.source_revision.outputs.value }} - run: ${{ matrix.builder || 'cargo' }} build --locked --release --package s2-cli --target ${{ matrix.target }} + run: ${{ matrix.builder || 'cargo' }} build --release --package s2-cli --target ${{ matrix.target }} - name: Verify source revision stamp shell: bash env: diff --git a/.github/workflows/release-plz.yml b/.github/workflows/release-plz.yml index ef380af4..e1961eb5 100644 --- a/.github/workflows/release-plz.yml +++ b/.github/workflows/release-plz.yml @@ -27,13 +27,45 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 + token: ${{ steps.generate-token.outputs.token }} - name: Install Rust uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable - name: Run release-plz + id: release-plz uses: release-plz/action@1528104d2ca23787631a1c1f022abb64b34c1e11 # v0.5.128 with: command: release-pr env: GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} + + - name: Refresh simulator lockfile in release PR + env: + GH_TOKEN: ${{ steps.generate-token.outputs.token }} + RELEASE_PR: ${{ steps.release-plz.outputs.pr }} + run: | + set -euo pipefail + + if [[ -z "$RELEASE_PR" || "$RELEASE_PR" == "null" ]]; then + exit 0 + fi + + pr_number="$(jq -r '.number // empty' <<<"$RELEASE_PR")" + if [[ -z "$pr_number" ]]; then + exit 0 + fi + + gh pr checkout "$pr_number" + cargo metadata --manifest-path sim/Cargo.toml --format-version 1 >/dev/null + cargo metadata --locked --manifest-path sim/Cargo.toml --format-version 1 >/dev/null + + if git diff --quiet -- sim/Cargo.lock; then + exit 0 + fi + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add sim/Cargo.lock + git commit -m "chore: refresh simulator lockfile" + git push diff --git a/.github/workflows/rust-dependency-cooldown-tests.yml b/.github/workflows/rust-dependency-cooldown-tests.yml deleted file mode 100644 index 76f56ecc..00000000 --- a/.github/workflows/rust-dependency-cooldown-tests.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Rust dependency cooldown tests - -on: - pull_request: - paths: - - .github/actions/rust-dependency-cooldown/** - - .github/workflows/rust-dependency-cooldown-tests.yml - push: - branches: [main] - paths: - - .github/actions/rust-dependency-cooldown/** - - .github/workflows/rust-dependency-cooldown-tests.yml - -permissions: - contents: read - -jobs: - test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - run: python3 -m unittest discover -s .github/actions/rust-dependency-cooldown -p 'test_*.py' -v diff --git a/.github/workflows/rust-dependency-cooldown.yml b/.github/workflows/rust-dependency-cooldown.yml deleted file mode 100644 index 067b8edc..00000000 --- a/.github/workflows/rust-dependency-cooldown.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Rust dependency cooldown gate - -on: - workflow_call: - -permissions: - contents: read - -jobs: - rust-dependency-cooldown: - name: Rust dependency cooldown gate - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Checkout repository - if: github.event_name == 'pull_request' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - fetch-depth: 2 - persist-credentials: false - - name: Check Rust dependency cooldown - if: github.event_name == 'pull_request' - uses: $/.github/actions/rust-dependency-cooldown - with: - base-sha: HEAD^1 diff --git a/AGENTS.md b/AGENTS.md index e2ad9644..0c2419c1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,12 +3,3 @@ - Formatting: run `just fmt` - Tests: run `just test` - PR title + description become the squashed commit message at merge time; use conventional commit format - -## Cargo Dependency Safety - -- Use `--locked` for Cargo commands that build, check, test, run, document, fetch, or read metadata. -- The simulator is temporarily exempt until its separate lockfile is regenerated. -- Use `cargo +nightly add`, `cargo +nightly update`, `cargo +nightly remove`, or `cargo +nightly generate-lockfile` for dependency changes. The repository Cargo configuration applies the publication cooldown. -- Do not use the stable forms of these dependency commands. -- Do not edit dependency declarations or `Cargo.lock` directly. -- Do not install Cargo tools as part of a coding task. diff --git a/Cargo.lock b/Cargo.lock index de3da95d..39735717 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,7 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ "crypto-common 0.2.2", - "inout", + "inout 0.2.2", ] [[package]] @@ -39,36 +39,59 @@ dependencies = [ [[package]] name = "aes" -version = "0.9.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cipher", + "cfg-if", + "cipher 0.4.4", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" +dependencies = [ + "cipher 0.5.2", "cpubits", - "cpufeatures 0.3.1", - "zeroize", + "cpufeatures 0.3.0", ] [[package]] name = "aes-gcm" -version = "0.11.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f" +checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" dependencies = [ "aead", - "aes", - "cipher", + "aes 0.9.2", + "cipher 0.5.2", "ctr", - "ctutils", "ghash", + "subtle", "zeroize", ] [[package]] -name = "aho-corasick" -version = "1.1.5" +name = "ahash" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -87,9 +110,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" dependencies = [ "libc", ] @@ -152,9 +175,9 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "apple-native-keyring-store" -version = "1.0.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b350bfd03649e07aa05c0a81b3e15934374e585c98204a57e20b9d49f49bb9a" +checksum = "797f94b6a53d7d10b56dc18290e0d40a2158352f108bb4ff32350825081a9f29" dependencies = [ "keyring-core", "log", @@ -176,6 +199,12 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + [[package]] name = "arrayvec" version = "0.7.8" @@ -194,7 +223,7 @@ dependencies = [ "nom", "num-traits", "rusticata-macros", - "thiserror 2.0.20", + "thiserror 2.0.19", "time", ] @@ -284,9 +313,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.46" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f10dafd0c8d2e51ae9a748805777613ed0bbe17bf586b76c8311f45c020a32f" +checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8" dependencies = [ "compression-codecs", "compression-core", @@ -414,21 +443,15 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.92" +version = "0.1.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] -[[package]] -name = "asyncband" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2d85fd3d291fabcc40c7232c92c280ec1754fd7b5d7ea769f143222143e179a" - [[package]] name = "atomic" version = "0.6.1" @@ -452,9 +475,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.12.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" +checksum = "1b180a3c8b55960db3426d8964b8745e652466a1a49fe1a2eda828046d30b5e4" dependencies = [ "aws-credential-types", "aws-runtime", @@ -495,9 +518,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.18.1" +version = "1.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" +checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -506,9 +529,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.45.0" +version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" +checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" dependencies = [ "cc", "cmake", @@ -519,9 +542,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.9.2" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef47857a1d4488b528f4a5d5715fa7c3300820897824152234d3fa22b1426657" +checksum = "c9007227e10b5fed2f3e0a2beff489211e2b5604c400b7a9d5d81ca9d64c24bb" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -544,9 +567,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.109.0" +version = "1.105.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3cfe74df5d9ad2fedd691973ad3521ebf4f27a3c68c792556686aedb5519bab" +checksum = "6ffd0fbe7873cb548a7aa60f9573c268fff94155397fd4f14dc9f1ecaaab8516" dependencies = [ "arc-swap", "aws-credential-types", @@ -570,9 +593,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.111.0" +version = "1.107.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b0ec31ed6191bd11350aae4b2004198f2db21350cb0a20c57e0a92e55dd161" +checksum = "175763eb222a46377df7aa257a3bca980ab3e96703fefc8f4d0b8da6ad2e254c" dependencies = [ "arc-swap", "aws-credential-types", @@ -596,9 +619,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.114.0" +version = "1.110.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef45745026107ec30c4ef86bd8ae4b002e7e5f6a86e4225240bdf6b06a0b944a" +checksum = "dd8b14781dfbff48984017d57167b6ea0b6471c6920ec52b44a2677c7feb3c13" dependencies = [ "arc-swap", "aws-credential-types", @@ -634,11 +657,11 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.13.0", "http 0.2.12", "http 1.5.0", "percent-encoding", - "sha2", + "sha2 0.11.0", "time", "tracing", ] @@ -677,9 +700,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.4.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebfd138fac0337cee7516c352757ea73b9f2266e57d0bcb5bc70e9547e45aef1" +checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -734,9 +757,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.14.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b82e438d30e02a825d363bd639a9efaed68a8089d86101054b0081e7e0d3e606" +checksum = "07505b34e8f4b3591a4fa69e9792b52289b95488dbbc68c3c0075b7bedb245e1" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -760,9 +783,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.16.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223" +checksum = "3b98f2e1fd67ec06618f9c291e5e495a468e60519e44c9c1979cd0521f3affdb" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -800,9 +823,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.6.3" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9" +checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8" dependencies = [ "base64-simd", "bytes", @@ -835,9 +858,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.6.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" +checksum = "eec1cd5469f328c782dc3e33d4153cf118a54e33cbb3356d60d16f89883e1f94" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -954,7 +977,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide 0.8.9", + "miniz_oxide", "object", "rustc-demangle", "windows-link 0.2.1", @@ -975,12 +998,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" - [[package]] name = "base64-simd" version = "0.8.0" @@ -1032,30 +1049,25 @@ dependencies = [ [[package]] name = "bitflags" -version = "1.3.2" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" dependencies = [ "serde_core", ] [[package]] name = "blake3" -version = "1.8.7" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" dependencies = [ + "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", ] [[package]] @@ -1078,18 +1090,18 @@ dependencies = [ [[package]] name = "block-padding" -version = "0.4.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "hybrid-array", + "generic-array", ] [[package]] name = "blocking" -version = "1.7.0" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" dependencies = [ "async-channel", "async-task", @@ -1100,13 +1112,13 @@ dependencies = [ [[package]] name = "bollard" -version = "0.21.1" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbe8358268799ebb3e4df23cb9d47f4c72bbc4f5247e2fa6a1bf7b6c0baea220" +checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4" dependencies = [ "async-stream", - "base64 0.22.1", - "bitflags 2.13.2", + "base64", + "bitflags", "bollard-buildkit-proto", "bollard-stubs", "bytes", @@ -1124,7 +1136,7 @@ dependencies = [ "log", "num", "pin-project-lite", - "rand 0.10.2", + "rand 0.9.5", "rustls", "rustls-native-certs", "rustls-pki-types", @@ -1132,7 +1144,7 @@ dependencies = [ "serde_derive", "serde_json", "serde_urlencoded", - "thiserror 2.0.20", + "thiserror 2.0.19", "time", "tokio", "tokio-stream", @@ -1145,23 +1157,24 @@ dependencies = [ [[package]] name = "bollard-buildkit-proto" -version = "0.8.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5c97450e79c7c565302dd92e86b08823b47550fcb4fc5ce910194d1b087a1a3" +checksum = "85a885520bf6249ab931a764ffdb87b0ceef48e6e7d807cfdb21b751e086e1ad" dependencies = [ "prost", "prost-types", "tonic", "tonic-prost", + "ureq", ] [[package]] name = "bollard-stubs" -version = "1.53.1-rc.29.3.1" +version = "1.52.1-rc.29.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce412eb6f7096743011dc3cb5c674caeb24ced61d8c498fe07cf7998a4fea889" +checksum = "0f0a8ca8799131c1837d1282c3f81f31e76ceb0ce426e04a7fe1ccee3287c066" dependencies = [ - "base64 0.22.1", + "base64", "bollard-buildkit-proto", "bytes", "prost", @@ -1182,9 +1195,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.13.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530" dependencies = [ "memchr", "regex-automata", @@ -1199,9 +1212,9 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytecheck" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26333eeac754f0ad8a6bcd0eb0ac012156302e4e16b852b72ee399aea4f12c29" +checksum = "0caa33a2c0edca0419d15ac723dff03f1956f7978329b1e3b5fdaaaed9d3ca8b" dependencies = [ "bytecheck_derive", "ptr_meta", @@ -1211,13 +1224,13 @@ dependencies = [ [[package]] name = "bytecheck_derive" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46d07918caa9eeaaf06b7873925c53a61daac173539b4f7715090745e44e4e69" +checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", ] [[package]] @@ -1259,9 +1272,9 @@ dependencies = [ [[package]] name = "bytesize" -version = "2.7.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7354288c522e7e980fafd2075d63d1285794c3a6a16cdd492f189ea406e5f18b" +checksum = "351a3e803ee3c6eaeee6b00076b767514b37c32a73d326c3ec7abddb7d6c3493" [[package]] name = "castaway" @@ -1274,18 +1287,18 @@ dependencies = [ [[package]] name = "cbc" -version = "0.2.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" dependencies = [ - "cipher", + "cipher 0.4.4", ] [[package]] name = "cc" -version = "1.4.5" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" dependencies = [ "find-msvc-tools", "jobserver", @@ -1307,12 +1320,12 @@ checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" -version = "0.10.2" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", "rand_core 0.10.1", ] @@ -1330,6 +1343,16 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout 0.1.4", +] + [[package]] name = "cipher" version = "0.5.2" @@ -1338,14 +1361,14 @@ checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ "block-buffer 0.12.1", "crypto-common 0.2.2", - "inout", + "inout 0.2.2", ] [[package]] name = "clap" -version = "4.6.6" +version = "4.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" dependencies = [ "clap_builder", "clap_derive", @@ -1353,9 +1376,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.6" +version = "4.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" dependencies = [ "anstream", "anstyle", @@ -1372,7 +1395,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -1396,6 +1419,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "cmsketch" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7ee2cfacbd29706479902b06d75ad8f1362900836aa32799eabc7e004bfd854" + [[package]] name = "color-print" version = "0.3.7" @@ -1429,14 +1458,14 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] name = "combine" -version = "4.6.8" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ "bytes", "memchr", @@ -1459,22 +1488,22 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.41" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58a6d0db8759036a783bc7c3f7a07f8cef3bf9470eb1db3bc86e8bcd1c5d0fe8" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" dependencies = [ "compression-core", "flate2", "memchr", - "zstd 0.14.0", - "zstd-safe 8.0.0", + "zstd", + "zstd-safe", ] [[package]] name = "compression-core" -version = "0.4.33" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" [[package]] name = "concurrent-queue" @@ -1500,16 +1529,16 @@ dependencies = [ "serde-untagged", "serde_core", "serde_json", - "toml 1.1.6+spec-1.1.0", + "toml 1.1.4+spec-1.1.0", "winnow 1.0.4", "yaml-rust2", ] [[package]] name = "console" -version = "0.16.6" +version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3" +checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" dependencies = [ "encode_unicode", "libc", @@ -1585,12 +1614,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "core_detect" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" - [[package]] name = "cpubits" version = "0.1.1" @@ -1608,9 +1631,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] @@ -1627,18 +1650,18 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-epoch" -version = "0.9.21" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] @@ -1655,9 +1678,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.23" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" [[package]] name = "crunchy" @@ -1692,7 +1715,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" dependencies = [ - "cipher", + "cipher 0.5.2", ] [[package]] @@ -1716,12 +1739,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.24.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core 0.24.1", - "darling_macro 0.24.1", + "darling_core 0.23.0", + "darling_macro 0.23.0", ] [[package]] @@ -1739,15 +1762,15 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.24.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ "ident_case", "proc-macro2", "quote", "strsim", - "syn 3.0.5", + "syn 2.0.119", ] [[package]] @@ -1763,13 +1786,13 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.24.1" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core 0.24.1", + "darling_core 0.23.0", "quote", - "syn 3.0.5", + "syn 2.0.119", ] [[package]] @@ -1788,46 +1811,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.11.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" - -[[package]] -name = "datasketches" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" - -[[package]] -name = "defmt" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" -dependencies = [ - "bitflags 1.3.2", - "defmt-macros", -] - -[[package]] -name = "defmt-macros" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" -dependencies = [ - "defmt-parser", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "defmt-parser" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" -dependencies = [ - "thiserror 2.0.20", -] +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "der-parser" @@ -1866,6 +1852,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "crypto-common 0.1.7", + "subtle", ] [[package]] @@ -1882,9 +1869,9 @@ dependencies = [ [[package]] name = "dirs" -version = "7.0.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d57d423b3c82e89b9a24ca3091fee61f456a26edbd28d26c65906f4bc1dcd8f" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ "dirs-sys", ] @@ -1898,7 +1885,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1909,7 +1896,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -1927,7 +1914,7 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d" dependencies = [ - "base64 0.22.1", + "base64", "serde", "serde_json", ] @@ -1952,7 +1939,7 @@ checksum = "f88959de2d447fd3eddcf1909d1f19fe084e27a056a6904203dc5d8b9e771c1e" dependencies = [ "rust_decimal", "serde", - "thiserror 2.0.20", + "thiserror 2.0.19", "time", "winnow 0.6.26", ] @@ -1965,9 +1952,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" -version = "1.18.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" [[package]] name = "encode_unicode" @@ -1977,17 +1964,11 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "encoding_rs" -version = "0.8.41" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5ef0006ac9ab233c38522f5ae99cae3625151de8f706cacee1cba4b8e2832a" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ "cfg-if", - "core_detect", - "multiversion", - "multiversion_no_op", - "rustversion", - "scopeguard", - "simdutf8", ] [[package]] @@ -2062,7 +2043,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2097,11 +2078,10 @@ dependencies = [ [[package]] name = "eyre" -version = "0.6.14" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08309dbcc659c5549a24ddb9b27027640641b282ef5768267c7e675558986a3" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" dependencies = [ - "autocfg", "indenter", "once_cell", ] @@ -2134,6 +2114,18 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "faststr" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca7d44d22004409a61c393afb3369c8f7bb74abcae49fe249ee01dcc3002113" +dependencies = [ + "bytes", + "rkyv", + "serde", + "simdutf8", +] + [[package]] name = "ferroid" version = "2.0.0" @@ -2163,9 +2155,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.12" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "fixedbitset" @@ -2179,18 +2171,18 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.13.2", + "bitflags", "rustc_version", ] [[package]] name = "flate2" -version = "1.1.10" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", - "miniz_oxide 0.9.1", + "miniz_oxide", "zlib-rs", ] @@ -2232,18 +2224,18 @@ dependencies = [ [[package]] name = "foyer" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a59f42276891c0a4ce683fcda1aa1b4fd1065d68116c264e4bf49b9d318a0ed" +checksum = "3b0abc0b87814989efa711f9becd9f26969820e2d3905db27d10969c4bd45890" dependencies = [ "anyhow", - "asyncband", "equivalent", "foyer-common", "foyer-memory", "foyer-storage", "foyer-tokio", "futures-util", + "mea", "mixtrics", "pin-project", "serde", @@ -2252,9 +2244,9 @@ dependencies = [ [[package]] name = "foyer-common" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643b47d510032a01e5af70dca288b0c5ec531646c1a8392e793fb5b0c13bef30" +checksum = "a3db80d5dece93adb7ad709c84578794724a9cba342a7e566c3551c7ec626789" dependencies = [ "anyhow", "bincode", @@ -2279,21 +2271,21 @@ dependencies = [ [[package]] name = "foyer-memory" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae51f5089f3d9025ae77f17ea66d2489ac0f82fbb1d91462807bea174d486d82" +checksum = "db907f40a527ca2aa2f40a5f68b32ea58aa70f050cd233518e9ffd402cfba6ce" dependencies = [ "anyhow", - "asyncband", - "bitflags 2.13.2", - "datasketches", + "bitflags", + "cmsketch", "equivalent", "foyer-common", "foyer-intrusive-collections", "foyer-tokio", "futures-util", - "hashbrown 0.17.1", - "itertools 0.15.0", + "hashbrown 0.16.1", + "itertools 0.14.0", + "mea", "mixtrics", "parking_lot", "paste", @@ -2304,13 +2296,12 @@ dependencies = [ [[package]] name = "foyer-storage" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "118192f532ba013f0cdc607efc22324b9ed855baed185608af0daa986e835c6b" +checksum = "1983f1db3d0710e9c9d5fc116d9202dccd41a2d1e032572224f1aff5520aa958" dependencies = [ "allocator-api2", "anyhow", - "asyncband", "bytes", "core_affinity", "equivalent", @@ -2321,25 +2312,26 @@ dependencies = [ "fs4", "futures-core", "futures-util", - "hashbrown 0.17.1", + "hashbrown 0.16.1", "io-uring", - "itertools 0.15.0", + "itertools 0.14.0", "libc", "lz4", + "mea", "parking_lot", "pin-project", - "rand 0.10.2", + "rand 0.9.5", "serde", "tracing", "twox-hash", - "zstd 0.13.3", + "zstd", ] [[package]] name = "foyer-tokio" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6741d1133dfaab64d3f8a9290bbfed3685084add28f8b6ee7a6264aa4dc26918" +checksum = "f6577b05a7ffad0db555aedf00bfe52af818220fc4c1c3a7a12520896fc38627" dependencies = [ "tokio", ] @@ -2382,9 +2374,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" dependencies = [ "futures-channel", "futures-core", @@ -2397,9 +2389,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" dependencies = [ "futures-core", "futures-sink", @@ -2407,15 +2399,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" [[package]] name = "futures-executor" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" dependencies = [ "futures-core", "futures-task", @@ -2424,9 +2416,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" [[package]] name = "futures-lite" @@ -2443,26 +2435,26 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", ] [[package]] name = "futures-sink" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" [[package]] name = "futures-task" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" [[package]] name = "futures-timer" @@ -2472,9 +2464,9 @@ checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" dependencies = [ "futures-channel", "futures-core", @@ -2543,7 +2535,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" dependencies = [ "polyval", - "zeroize", ] [[package]] @@ -2572,9 +2563,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.19" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", @@ -2582,7 +2573,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.5.0", - "indexmap 2.14.2", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", @@ -2616,6 +2607,8 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ + "allocator-api2", + "equivalent", "foldhash 0.2.0", ] @@ -2653,9 +2646,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.5.3" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "hex" @@ -2665,11 +2658,20 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hkdf" -version = "0.13.0" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", ] [[package]] @@ -2734,9 +2736,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.5" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" dependencies = [ "bytes", "futures-core", @@ -2765,18 +2767,18 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" -version = "0.4.15" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" dependencies = [ "typenum", ] [[package]] name = "hyper" -version = "1.11.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" dependencies = [ "atomic-waker", "bytes", @@ -2843,7 +2845,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", "futures-util", @@ -2901,9 +2903,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", @@ -2915,43 +2917,22 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", - "serde", "tinystr", "writeable", "zerovec", ] -[[package]] -name = "icu_locale_fallback" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" -dependencies = [ - "icu_locale_core", - "icu_locale_fallback_data", - "icu_provider", - "potential_utf", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locale_fallback_data" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" - [[package]] name = "icu_normalizer" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -2963,17 +2944,16 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ - "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -2984,20 +2964,18 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.3.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", - "serde", - "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -3005,28 +2983,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "icu_segmenter" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82d07aafccd67af15d02512a6adf5896fbc5ed00f2e99b471d2efa14016db3db" -dependencies = [ - "icu_collections", - "icu_locale_fallback", - "icu_provider", - "icu_segmenter_data", - "potential_utf", - "smallvec", - "utf8_iter", - "zerovec", -] - -[[package]] -name = "icu_segmenter_data" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae293c039020f9ec10710af98d29ce6aa2051486638b49c9a6409f3b4a9e98ad" - [[package]] name = "ident_case" version = "1.0.1" @@ -3073,9 +3029,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.2" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -3102,32 +3058,41 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "inout" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" dependencies = [ - "block-padding", "hybrid-array", ] [[package]] name = "io-uring" -version = "0.7.15" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" +checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0" dependencies = [ - "bitflags 2.13.2", + "bitflags", "cfg-if", "libc", ] [[package]] name = "ipnet" -version = "2.12.2" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "is_ci" @@ -3165,59 +3130,6 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "jiff" -version = "0.2.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" -dependencies = [ - "defmt", - "jiff-core", - "jiff-static", - "jiff-tzdb-platform", - "log", - "portable-atomic", - "portable-atomic-util", - "serde_core", - "windows-link 0.2.1", -] - -[[package]] -name = "jiff-core" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" -dependencies = [ - "defmt", -] - -[[package]] -name = "jiff-static" -version = "0.2.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" -dependencies = [ - "jiff-core", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "jiff-tzdb" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" - -[[package]] -name = "jiff-tzdb-platform" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" -dependencies = [ - "jiff-tzdb", -] - [[package]] name = "jni" version = "0.22.4" @@ -3230,7 +3142,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.20", + "thiserror 2.0.19", "walkdir", "windows-link 0.2.1", ] @@ -3279,9 +3191,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.105" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" dependencies = [ "cfg-if", "futures-util", @@ -3301,9 +3213,9 @@ dependencies = [ [[package]] name = "json_to_table" -version = "0.14.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec64c9908ffa97b6ef6044d96c9f56de1c81643d3b9fbc2823e8847ab11b467" +checksum = "1f91246cf42b255a705436e360ca5894e376a50fdf696e4e60cb1bb6dd648e21" dependencies = [ "serde_json", "tabled", @@ -3311,9 +3223,9 @@ dependencies = [ [[package]] name = "keyring" -version = "4.2.0" +version = "4.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2270074a3d26bcac93c1dc5d2845eb4c089e8d761ccf6e0ea266a16004640627" +checksum = "72585bb6cc9bc370d1d545b7e23fcce71dfd4461c5e15275e3cf51bdfd9a980a" dependencies = [ "apple-native-keyring-store", "keyring-core", @@ -3344,9 +3256,9 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libredox" -version = "0.1.23" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed" +checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" dependencies = [ "libc", ] @@ -3365,9 +3277,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" @@ -3380,15 +3292,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.34" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" [[package]] name = "lru" -version = "0.18.4" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" +checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" dependencies = [ "hashbrown 0.17.1", ] @@ -3452,6 +3364,15 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "mea" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31fc7d159de0085ab6dd7ff145a9819442cfd3d098f783263120503c3f3e58b0" +dependencies = [ + "slab", +] + [[package]] name = "memchr" version = "2.8.3" @@ -3514,15 +3435,6 @@ name = "miniz_oxide" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", -] - -[[package]] -name = "miniz_oxide" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -3530,9 +3442,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.3" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", "wasi", @@ -3555,33 +3467,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" -[[package]] -name = "multiversion" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ca4bea16ffc3f443cf7d866912118196bfef4c6a1556ca00f9f9b00bb43f7c" -dependencies = [ - "multiversion-macros", -] - -[[package]] -name = "multiversion-macros" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d416831a7317ef4b08bee00b69cbbb9c8763da7959a7026244d6266869f9c83" -dependencies = [ - "proc-macro2", - "quote", - "rustversion", - "syn 3.0.5", -] - -[[package]] -name = "multiversion_no_op" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" - [[package]] name = "munge" version = "0.4.7" @@ -3608,7 +3493,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags 2.13.2", + "bitflags", "cfg-if", "cfg_aliases", "libc", @@ -3645,7 +3530,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3689,9 +3574,9 @@ checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" -version = "0.1.47" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ "num-traits", ] @@ -3742,7 +3627,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags 2.13.2", + "bitflags", ] [[package]] @@ -3772,7 +3657,7 @@ checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50" dependencies = [ "async-trait", "aws-lc-rs", - "base64 0.22.1", + "base64", "bytes", "chrono", "crc-fast", @@ -3796,7 +3681,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.20", + "thiserror 2.0.19", "tokio", "tracing", "url", @@ -3891,9 +3776,9 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "owo-colors" -version = "4.4.0" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8" +checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" [[package]] name = "papergrid" @@ -3937,9 +3822,9 @@ dependencies = [ [[package]] name = "parse-display" -version = "0.11.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e78deb158fb1d73b29efb4b7e9b9860b78059c670de06bd28df8d0b458ded0eb" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" dependencies = [ "parse-display-derive", "regex", @@ -3948,9 +3833,9 @@ dependencies = [ [[package]] name = "parse-display-derive" -version = "0.11.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e95a50d1084dab562913062c4c34bb204b68fc6ec38a1395909ff5aaaf4f10a" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" dependencies = [ "proc-macro2", "quote", @@ -3997,11 +3882,11 @@ dependencies = [ [[package]] name = "pem" -version = "4.0.0" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64 0.23.1", + "base64", "serde_core", ] @@ -4013,9 +3898,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.9.1" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d45aeb61b4bf818e12d4205f2466f8c4748f85f4fce0146d1c03d69d753f0ad" +checksum = "7df728be843c7070fab6ab7c328c4e9e9d78e23bf749c0669c86ee7ebfa050a2" dependencies = [ "memchr", "ucd-trie", @@ -4023,9 +3908,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.9.1" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89cc5a242e25ed4e7704d0be240f2cfbe20a8c27e7e252d94835be93d92dc39f" +checksum = "9e2dd6fc3b26b3462ee188aac870f5a41d398f1cd5e2408d16531bd71c9591fd" dependencies = [ "pest", "pest_generator", @@ -4033,9 +3918,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.9.1" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7abf21475cc3820fe4b2ca2dc2142902f67a02189f3b5b3a229f4febc01a43e5" +checksum = "6a7a9205cfb6f596a9e8b689c0a15f9ceb7a1aafae7aaf788150ac65b29975b6" dependencies = [ "pest", "pest_meta", @@ -4046,9 +3931,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.9.1" +version = "2.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adba4db388f687393c18c51348d44a41d870ca9df71a2c98172ea3035dc6936e" +checksum = "85abd351c0de1e8384fc791a0737111a350394937e92b956b743dac12429f57c" dependencies = [ "pest", ] @@ -4061,7 +3946,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap 2.14.2", + "indexmap 2.14.0", ] [[package]] @@ -4109,9 +3994,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "polling" @@ -4134,34 +4019,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" dependencies = [ "cpubits", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", "universal-hash", - "zeroize", ] [[package]] name = "portable-atomic" -version = "1.15.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" - -[[package]] -name = "portable-atomic-util" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" -dependencies = [ - "portable-atomic", -] +checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" [[package]] name = "potential_utf" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ - "serde_core", - "writeable", "zerovec", ] @@ -4226,7 +4099,29 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.15+spec-1.1.0", + "toml_edit 0.25.13+spec-1.1.0", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -4257,7 +4152,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags 2.13.2", + "bitflags", "hex", "procfs-core", "rustix 0.38.44", @@ -4269,7 +4164,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags 2.13.2", + "bitflags", "hex", ] @@ -4287,7 +4182,7 @@ dependencies = [ "parking_lot", "procfs", "protobuf", - "thiserror 2.0.20", + "thiserror 2.0.19", ] [[package]] @@ -4298,7 +4193,7 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", "bit-vec 0.8.0", - "bitflags 2.13.2", + "bitflags", "num-traits", "rand 0.9.5", "rand_chacha", @@ -4382,22 +4277,22 @@ dependencies = [ [[package]] name = "ptr_meta" -version = "0.3.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743da816b98c921cdbe8628ef7381b76f25ecf4da599fc80aca90eae7ef70cc0" +checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" dependencies = [ "ptr_meta_derive", ] [[package]] name = "ptr_meta_derive" -version = "0.3.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c8d9ca532f185d5d4db7a7c9d51420b452168ea1c2b913953281bd6fe1fcbd0" +checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", ] [[package]] @@ -4430,7 +4325,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.20", + "thiserror 2.0.19", "tokio", "tracing", "web-time", @@ -4438,9 +4333,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.17" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "aws-lc-rs", "bytes", @@ -4453,7 +4348,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.20", + "thiserror 2.0.19", "tinyvec", "tracing", "web-time", @@ -4470,7 +4365,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4496,9 +4391,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rancor" -version = "0.1.3" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b534442d0fcdb55d66f373d9cac6d33b6293a2335bc2136dbd06ce0e87d2572" +checksum = "daff8b7b3ccf5f7ba270b3e7a0a4d4c701c5797e38dec27c7e2c3dbb830fed1c" dependencies = [ "ptr_meta", ] @@ -4578,9 +4473,9 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.14.10" +version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055" dependencies = [ "aws-lc-rs", "pem", @@ -4596,7 +4491,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.13.2", + "bitflags", ] [[package]] @@ -4607,27 +4502,27 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.20", + "thiserror 2.0.19", ] [[package]] name = "ref-cast" -version = "1.0.27" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.27" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -4644,9 +4539,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.18" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" dependencies = [ "aho-corasick", "memchr", @@ -4682,11 +4577,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.5" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64 0.23.1", + "base64", "bytes", "futures-core", "futures-util", @@ -4737,14 +4632,14 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.8.18" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9776093b7ca170454ab1406954f7b7d97a57c51dc6c0642957fb2ef25c2d399" +checksum = "815cc8a37159a463064825246cadb07961e25cd9885908606f6d08a98d8f8874" dependencies = [ "bytecheck", "bytes", "hashbrown 0.17.1", - "indexmap 2.14.2", + "indexmap 2.14.0", "munge", "ptr_meta", "rancor", @@ -4756,13 +4651,13 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.8.18" +version = "0.8.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c25ef604ac7dd839d44d64648952ea23c97866f124ff671b0ed2cf3ad9bb06e" +checksum = "c0ed1a78a1b19d184b0daa629dd9a024573173ec7d485b287cb369fb3607cc1c" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", ] [[package]] @@ -4771,7 +4666,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81116b9531d61eabc41aeb228e4b6b2435bcca3233b98cf3b3077d4e6e9debb3" dependencies = [ - "bitflags 2.13.2", + "bitflags", "once_cell", "serde", "serde_derive", @@ -4792,9 +4687,9 @@ dependencies = [ [[package]] name = "rstest" -version = "0.27.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "948203e6d13b83e51a90d7cf236dd58a0065f23f4b902f00f306e7eb2bd09b9c" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" dependencies = [ "futures-timer", "futures-util", @@ -4803,9 +4698,9 @@ dependencies = [ [[package]] name = "rstest_macros" -version = "0.27.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d92eaf7b6e51e12471d71ddc72608e7151573de44099aacfbb1128177c0da4" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" dependencies = [ "cfg-if", "glob", @@ -4821,12 +4716,12 @@ dependencies = [ [[package]] name = "rtoolbox" -version = "0.0.6" +version = "0.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1efe12a1469752d0e6ff5ebec0b6ef4924cc5c4c71046b0ec730040535819d" +checksum = "50a0e551c1e27e1731aba276dbeaeac73f53c7cd34d1bda485d02bd1e0f36844" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4841,9 +4736,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.43.0" +version = "1.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7653272e75dcac41dc199fbea6f5797633994fafd339943c06c9af16bf29cd3a" +checksum = "be2a24f50780bc85f09cc6ac299bdf1424302742d77221106859c9d8b102126a" dependencies = [ "arrayvec", "num-traits", @@ -4885,7 +4780,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.13.2", + "bitflags", "errno", "libc", "linux-raw-sys 0.4.15", @@ -4898,18 +4793,18 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.13.2", + "bitflags", "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" dependencies = [ "aws-lc-rs", "log", @@ -4961,7 +4856,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4972,9 +4867,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", @@ -5008,7 +4903,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s2-api" -version = "0.32.0" +version = "0.31.1" dependencies = [ "axum", "base64ct", @@ -5028,17 +4923,18 @@ dependencies = [ "s2-common", "serde", "serde_json", + "sonic-rs", "strum", - "thiserror 2.0.20", + "thiserror 2.0.19", "time", "tokio-util", "utoipa", - "zstd 0.14.0", + "zstd", ] [[package]] name = "s2-cli" -version = "0.43.0" +version = "0.42.7" dependencies = [ "assert_cmd", "async-stream", @@ -5049,15 +4945,11 @@ dependencies = [ "clap", "color-print", "colored", - "compact_str", "config", "dirs", "fs2", "futures", - "http-body-util", "humantime", - "hyper", - "hyper-util", "indicatif", "json_to_table", "keyring", @@ -5080,16 +4972,16 @@ dependencies = [ "serde", "serde_json", "serial_test", - "sha2", + "sha2 0.11.0", "strum", "tabled", "tempfile", "terminal_size", - "thiserror 2.0.20", + "thiserror 2.0.19", "tikv-jemallocator", "tokio", "tokio-stream", - "toml 1.1.6+spec-1.1.0", + "toml 1.1.4+spec-1.1.0", "tracing", "tracing-subscriber", "uuid", @@ -5099,7 +4991,7 @@ dependencies = [ [[package]] name = "s2-common" -version = "0.42.0" +version = "0.41.1" dependencies = [ "axum", "base64ct", @@ -5116,14 +5008,14 @@ dependencies = [ "serde", "serde_json", "strum", - "thiserror 2.0.20", + "thiserror 2.0.19", "time", "utoipa", ] [[package]] name = "s2-lite" -version = "0.43.0" +version = "0.42.7" dependencies = [ "async-stream", "async-trait", @@ -5138,7 +5030,7 @@ dependencies = [ "eyre", "futures", "http 1.5.0", - "indexmap 2.14.2", + "indexmap 2.14.0", "itertools 0.15.0", "parking_lot", "prometheus", @@ -5156,13 +5048,13 @@ dependencies = [ "serde_json", "slatedb", "strum", - "thiserror 2.0.20", + "thiserror 2.0.19", "tikv-jemallocator", "time", "tokio", "tokio-util", "tower", - "tower-http 0.7.1", + "tower-http 0.7.0", "tracing", "tracing-subscriber", "utoipa", @@ -5171,9 +5063,8 @@ dependencies = [ [[package]] name = "s2-resource-spec" -version = "0.3.0" +version = "0.2.2" dependencies = [ - "compact_str", "humantime", "s2-common", "schemars 1.2.2", @@ -5183,17 +5074,15 @@ dependencies = [ [[package]] name = "s2-sdk" -version = "0.35.0" +version = "0.34.3" dependencies = [ "assert_matches", "async-compression", "async-stream", "async-trait", "bytes", - "compact_str", "futures-core", "futures-util", - "h2", "http 1.5.0", "http-body 1.1.0", "http-body-util", @@ -5213,7 +5102,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "test-context", - "thiserror 2.0.20", + "thiserror 2.0.19", "time", "tokio", "tokio-muxt", @@ -5227,7 +5116,7 @@ dependencies = [ [[package]] name = "s2-storage" -version = "0.2.6" +version = "0.2.4" dependencies = [ "aegis", "aes-gcm", @@ -5239,17 +5128,17 @@ dependencies = [ "s2-common", "secrecy", "serde", - "thiserror 2.0.20", + "thiserror 2.0.19", ] [[package]] name = "s2-testcontainers" -version = "0.43.0" +version = "0.42.7" dependencies = [ "reqwest", "s2-sdk", "testcontainers", - "thiserror 2.0.20", + "thiserror 2.0.19", "tokio", ] @@ -5305,7 +5194,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -5325,20 +5214,20 @@ dependencies = [ [[package]] name = "secret-service" -version = "5.2.0" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5107b24b91445dd2aa449a258a1807b63240942157292354dc5bfdbeb8bc6db8" +checksum = "9a62d7f86047af0077255a29494136b9aaaf697c76ff70b8e49cded4e2623c14" dependencies = [ - "aes", + "aes 0.8.4", "cbc", "futures-util", - "getrandom 0.4.3", + "generic-array", + "getrandom 0.2.17", "hkdf", - "hybrid-array", "num", "once_cell", "serde", - "sha2", + "sha2 0.10.9", "zbus", ] @@ -5348,7 +5237,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.13.2", + "bitflags", "core-foundation", "core-foundation-sys", "libc", @@ -5421,7 +5310,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -5432,7 +5321,7 @@ checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -5441,7 +5330,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ - "indexmap 2.14.2", + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -5468,7 +5357,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -5503,17 +5392,16 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.23.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" dependencies = [ - "base64 0.23.1", + "base64", "bs58", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.14.2", - "jiff", + "indexmap 2.14.0", "schemars 0.9.0", "schemars 1.2.2", "serde_core", @@ -5524,14 +5412,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.23.0" +version = "3.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" dependencies = [ - "darling 0.24.1", + "darling 0.23.0", "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", ] [[package]] @@ -5540,7 +5428,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.14.2", + "indexmap 2.14.0", "itoa", "ryu", "serde", @@ -5569,7 +5457,7 @@ checksum = "a22144e767da4ddd8416dbf383700542ffd8a5dc493dfecedfe1fe3ad03c98ae" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -5583,6 +5471,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha2" version = "0.11.0" @@ -5590,7 +5489,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", "digest 0.11.3", ] @@ -5655,15 +5554,15 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "slatedb" -version = "0.16.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb40332f41e231926df3a0ef742c05316b43368ba4b520433d2a1eba1ab00f0f" +checksum = "35ca56b01922b15aa69fe3abb62cadc985d86032c9647e4606e211c4da751a76" dependencies = [ "async-channel", "async-trait", "atomic", "backon", - "bitflags 2.13.2", + "bitflags", "bytes", "chrono", "crc32fast", @@ -5697,14 +5596,14 @@ dependencies = [ "url", "uuid", "walkdir", - "zstd 0.13.3", + "zstd", ] [[package]] name = "slatedb-common" -version = "0.16.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b77f238f348e95e1653a5235f880ac703011b82582bd556f2aba405abf8180e" +checksum = "0aa8de522ff46a0f9b5a66f45e650d125421d4d91990933591092f9d010c40d1" dependencies = [ "chrono", "log", @@ -5718,9 +5617,9 @@ dependencies = [ [[package]] name = "slatedb-txn-obj" -version = "0.16.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad63b6c250219e26d71f497820f970274c301f896366daa56915f2a05df0e0a" +checksum = "d85fd9c0c86dd4954524fa8238d0548bd80f4a9a66983cbde3728402d339993e" dependencies = [ "async-trait", "bytes", @@ -5741,9 +5640,9 @@ checksum = "88414a5ca1f85d82cc34471e975f0f74f6aa54c40f062efa42c0080e7f763f81" [[package]] name = "smallvec" -version = "1.16.0" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" @@ -5761,6 +5660,45 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45e14297decde697ddf377c25752aead0927d5cfc89c2684d2af96901a4ceeea" +[[package]] +name = "sonic-number" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3775c3390edf958191f1ab1e8c5c188907feebd0f3ce1604cb621f72961dbf32" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "sonic-rs" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d971cc77a245ccf1756dbd1a87c3e7f709c0191464096510d43eec056d0f2c4f" +dependencies = [ + "ahash", + "bumpalo", + "bytes", + "cfg-if", + "faststr", + "itoa", + "ref-cast", + "serde", + "simdutf8", + "sonic-number", + "sonic-simd", + "thiserror 2.0.19", + "zmij", +] + +[[package]] +name = "sonic-simd" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f99e664ecd2d85a68c87e3c7a3cfe691f647ea9e835de984aba4d54a41f817d4" +dependencies = [ + "cfg-if", +] + [[package]] name = "spin" version = "0.10.1" @@ -5869,9 +5807,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.5" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" dependencies = [ "proc-macro2", "quote", @@ -5914,9 +5852,9 @@ dependencies = [ [[package]] name = "tabled" -version = "0.22.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2596a104db1900b943f97d793a6c99addca918c4525c999f751a0f17d472eb" +checksum = "b5dc662e6da844ad6e428ad16b57967c9d33c82e16bb1c258326c0c078605dff" dependencies = [ "papergrid", "tabled_derive", @@ -5925,11 +5863,12 @@ dependencies = [ [[package]] name = "tabled_derive" -version = "0.12.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dca1937322a1e892b1a65f6a6736183bb0a29d3b4234d1d53bc436dff9beb76" +checksum = "0ea5d1b13ca6cff1f9231ffd62f15eefd72543dab5e468735f1a456728a02846" dependencies = [ "heck 0.5.0", + "proc-macro-error2", "proc-macro2", "quote", "syn 2.0.119", @@ -5942,10 +5881,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5955,7 +5894,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5987,9 +5926,9 @@ dependencies = [ [[package]] name = "testcontainers" -version = "0.28.0" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2bbe381afaaa58ea610c5fc3ffb2184063a32b3e358a179f0b4865dd59934a" +checksum = "bfd5785b5483672915ed5fe3cddf9f546802779fc1eceff0a6fb7321fac81c1e" dependencies = [ "astral-tokio-tar", "async-trait", @@ -6009,7 +5948,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror 2.0.20", + "thiserror 2.0.19", "tokio", "tokio-stream", "tokio-util", @@ -6027,11 +5966,11 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.16.3" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b81c0cb5fce14f53e49c1d4da0c508334ff12040221bb8ab01b2dabd91d04b6e" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" dependencies = [ - "icu_segmenter", + "unicode-linebreak", "unicode-width 0.2.2", ] @@ -6046,11 +5985,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.20" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" dependencies = [ - "thiserror-impl 2.0.20", + "thiserror-impl 2.0.19", ] [[package]] @@ -6066,13 +6005,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.20" +version = "2.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -6106,9 +6045,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.55" +version = "0.3.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" dependencies = [ "deranged", "num-conv", @@ -6145,20 +6084,19 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.4" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", - "serde_core", "zerovec", ] [[package]] name = "tinyvec" -version = "1.13.2" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" dependencies = [ "tinyvec_macros", ] @@ -6194,7 +6132,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 3.0.3", ] [[package]] @@ -6209,9 +6147,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.5" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ "rustls", "tokio", @@ -6280,11 +6218,11 @@ dependencies = [ [[package]] name = "toml" -version = "1.1.6+spec-1.1.0" +version = "1.1.4+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "920602543f0911ab71da12c50d59701da54c196d1a2bf5cb4b75667f137a406a" +checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" dependencies = [ - "indexmap 2.14.2", + "indexmap 2.14.0", "serde_core", "serde_spanned 1.1.1", "toml_datetime 1.1.1+spec-1.1.0", @@ -6317,7 +6255,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.14.2", + "indexmap 2.14.0", "serde", "serde_spanned 0.6.9", "toml_datetime 0.6.11", @@ -6327,11 +6265,11 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.15+spec-1.1.0" +version = "0.25.13+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" dependencies = [ - "indexmap 2.14.2", + "indexmap 2.14.0", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "winnow 1.0.4", @@ -6366,7 +6304,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", - "base64 0.22.1", + "base64", "bytes", "h2", "http 1.5.0", @@ -6406,7 +6344,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "indexmap 2.14.2", + "indexmap 2.14.0", "pin-project-lite", "slab", "sync_wrapper", @@ -6423,7 +6361,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags 2.13.2", + "bitflags", "bytes", "futures-util", "http 1.5.0", @@ -6437,12 +6375,12 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.7.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c" +checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" dependencies = [ "async-compression", - "bitflags 2.13.2", + "bitflags", "bytes", "futures-core", "http 1.5.0", @@ -6539,9 +6477,12 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twox-hash" -version = "2.1.4" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" +dependencies = [ + "rand 0.10.2", +] [[package]] name = "typed-path" @@ -6610,6 +6551,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -6662,6 +6609,33 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64", + "log", + "percent-encoding", + "rustls", + "rustls-pki-types", + "ureq-proto", + "utf8-zero", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64", + "http 1.5.0", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -6681,6 +6655,12 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -6696,9 +6676,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "utoipa" version = "5.4.0" -source = "git+https://github.com/infiniteregrets/utoipa?rev=82bcb28a792ba9a0d29963827ec473823099fc94#82bcb28a792ba9a0d29963827ec473823099fc94" +source = "git+https://github.com/infiniteregrets/utoipa?rev=9cd181d40ac0a50551ebe1995a4d73e8685890d6#9cd181d40ac0a50551ebe1995a4d73e8685890d6" dependencies = [ - "indexmap 2.14.2", + "indexmap 2.14.0", "serde", "serde_json", "utoipa-gen", @@ -6707,7 +6687,7 @@ dependencies = [ [[package]] name = "utoipa-gen" version = "5.4.0" -source = "git+https://github.com/infiniteregrets/utoipa?rev=82bcb28a792ba9a0d29963827ec473823099fc94#82bcb28a792ba9a0d29963827ec473823099fc94" +source = "git+https://github.com/infiniteregrets/utoipa?rev=9cd181d40ac0a50551ebe1995a4d73e8685890d6#9cd181d40ac0a50551ebe1995a4d73e8685890d6" dependencies = [ "proc-macro2", "quote", @@ -6716,9 +6696,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.26.1" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -6790,9 +6770,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.128" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" dependencies = [ "cfg-if", "once_cell", @@ -6803,9 +6783,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.78" +version = "0.4.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" dependencies = [ "js-sys", "wasm-bindgen", @@ -6813,9 +6793,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.128" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6823,22 +6803,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.128" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.128" +version = "0.2.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" dependencies = [ "unicode-ident", ] @@ -6858,9 +6838,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.105" +version = "0.3.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" dependencies = [ "js-sys", "wasm-bindgen", @@ -6907,7 +6887,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -7203,9 +7183,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.4" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "x509-parser" @@ -7221,7 +7201,7 @@ dependencies = [ "nom", "oid-registry", "rusticata-macros", - "thiserror 2.0.20", + "thiserror 2.0.19", "time", ] @@ -7249,9 +7229,9 @@ checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "yaml-rust2" -version = "0.11.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b36710ce3a279cfce8465dbab826f161675a262950b922cb2c3663852dfe9eb0" +checksum = "631a50d867fafb7093e709d75aaee9e0e0d5deb934021fcea25ac2fe09edc51e" dependencies = [ "arraydeque", "encoding_rs", @@ -7299,9 +7279,9 @@ dependencies = [ [[package]] name = "zbus" -version = "5.19.0" +version = "5.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db4be7c075cb421e4b7ee645541604239bd243ba7c357511f4ff3a74b555907" +checksum = "fe18fb60dc696039e738717b76eaea21e7a4489bbb1885020b43c94236d7e98a" dependencies = [ "async-broadcast", "async-executor", @@ -7334,9 +7314,9 @@ dependencies = [ [[package]] name = "zbus-secret-service-keyring-store" -version = "1.0.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74801d001b9e7729adb4f1825b67b398185fed424749aa3d8bacf70417137d9a" +checksum = "4ccede190ba363386a24e8021c7f3848393976609ec9f5d1f8c6c09ef37075b4" dependencies = [ "keyring-core", "secret-service", @@ -7345,14 +7325,14 @@ dependencies = [ [[package]] name = "zbus_macros" -version = "5.19.0" +version = "5.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2990635d09ade6df1868f72f8cac69a876a90981e8bd3c40b1be413f8dc88f40" +checksum = "fe96480bed92df2b442a1a30df364e12d08eed03aeb061f2b8dc6afb2be91119" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", "zbus_names", "zvariant", "zvariant_utils", @@ -7369,29 +7349,20 @@ dependencies = [ "zvariant", ] -[[package]] -name = "zcheapstr" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1afec51604565183aeb5c54c20aeab286120d4e4460f7f76e3e8bb8c0d99473" -dependencies = [ - "serde", -] - [[package]] name = "zerocopy" -version = "0.8.57" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.57" +version = "0.8.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" dependencies = [ "proc-macro2", "quote", @@ -7427,23 +7398,21 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.5" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", "zerofrom", - "zerovec", ] [[package]] name = "zerovec" -version = "0.11.8" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ - "serde", "yoke", "zerofrom", "zerovec-derive", @@ -7451,13 +7420,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.6" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", ] [[package]] @@ -7468,7 +7437,7 @@ checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" dependencies = [ "crc32fast", "flate2", - "indexmap 2.14.2", + "indexmap 2.14.0", "memchr", "typed-path", "zopfli", @@ -7476,9 +7445,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.6.7" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" +checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5" [[package]] name = "zmij" @@ -7504,41 +7473,23 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe 7.3.0", -] - -[[package]] -name = "zstd" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e" -dependencies = [ - "zstd-safe 8.0.0", + "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.3.0" +version = "7.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-safe" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.1.0+zstd.1.5.7" +version = "2.0.16+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" dependencies = [ "cc", "pkg-config", @@ -7546,41 +7497,40 @@ dependencies = [ [[package]] name = "zvariant" -version = "5.15.0" +version = "5.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1d34c27cc6cdd1f458427519dd6b8612f7b7e3f7b9a0b2355d041dda9869147" +checksum = "bee2a0bcd2a907786a456fff45aaaaf54c9ba5f50b71ae9ec1a4edd200c94911" dependencies = [ "endi", "enumflags2", "serde", "winnow 1.0.4", - "zcheapstr", "zvariant_derive", "zvariant_utils", ] [[package]] name = "zvariant_derive" -version = "5.15.0" +version = "5.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "864155e69b4352db0c7f374917bf45d1e0c8d17659c8b3dbf9795f3673f8c497" +checksum = "38a708216a18780796770bfe3f4739c7c83a3e8f789b755534bbbc06e4e23e12" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 3.0.5", + "syn 2.0.119", "zvariant_utils", ] [[package]] name = "zvariant_utils" -version = "4.2.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad0294361a320b694a328460dc73add56c306150f5cb6bfafc44446120008a3" +checksum = "90cb9383f9b45290407a1258b202d3f8f01db719eb60b4e4055c6375af4fc7c7" dependencies = [ "proc-macro2", "quote", "serde", - "syn 3.0.5", + "syn 2.0.119", "winnow 1.0.4", ] diff --git a/Cargo.toml b/Cargo.toml index a76dfc09..037a28d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,14 +29,14 @@ axum-server = "0.8" base64ct = "1.8" blake3 = "1.8" bytes = "1" -bytesize = "2.7" +bytesize = "2.6" clap = "4.6" color-print = "0.3" colored = "3.1" compact_str = "0.10" config = "0.15" dashmap = "6.2" -dirs = "7.0" +dirs = "6.0" enumset = "1.1" eyre = "0.6" flate2 = "1.1" @@ -49,8 +49,8 @@ humantime = "2.4" indexmap = "2.14" indicatif = "0.18" itertools = "0.15" -json_to_table = "0.14" -keyring = "4.2" +json_to_table = "0.13" +keyring = "4.1" miette = "7.6" mime = "0.3" parking_lot = "0.12" @@ -63,24 +63,25 @@ rcgen = { version = "0.14", default-features = false, features = ["crypto", "pem reqwest = { version = "0.13", default-features = false, features = ["rustls"] } rkyv = "0.8" rpassword = "7.5" -rstest = "0.27" +rstest = "0.26" rustls = { version = "0.23", default-features = false, features = ["logging", "std", "tls12"] } -s2-api = { path = "api", version = "0.32" } -s2-common = { path = "common", version = "0.42" } -s2-lite = { path = "lite", version = "0.43" } -s2-resource-spec = { path = "resource-spec", version = "0.3" } -s2-sdk = { path = "sdk", version = "0.35" } +s2-api = { path = "api", version = "0.31" } +s2-common = { path = "common", version = "0.41" } +s2-lite = { path = "lite", version = "0.42" } +s2-resource-spec = { path = "resource-spec", version = "0.2" } +s2-sdk = { path = "sdk", version = "0.34" } s2-storage = { path = "storage", version = "0.2" } schemars = "1.2" secrecy = "0.10.3" semver = "1.0" serde = "1.0" serde_json = "1.0" -slatedb = "0.16.0" +slatedb = "0.15.0" +sonic-rs = "0.5" strum = "0.28" -tabled = "0.22" +tabled = "0.21" tempfile = "3.27" -testcontainers = "0.28" +testcontainers = "0.27" thiserror = "2.0" tikv-jemallocator = { version = "0.7", features = ["unprefixed_malloc_on_supported_platforms"] } time = "0.3" @@ -92,12 +93,12 @@ tower-http = "0.7" tracing = "0.1" tracing-subscriber = "0.3" utoipa = "=5.4" -uuid = "1.26" +uuid = "1.24" xxhash-rust = "0.8" -zstd = "0.14" +zstd = "0.13" [patch.crates-io] -utoipa = { git = "https://github.com/infiniteregrets/utoipa", rev = "82bcb28a792ba9a0d29963827ec473823099fc94" } +utoipa = { git = "https://github.com/infiniteregrets/utoipa", rev = "9cd181d40ac0a50551ebe1995a4d73e8685890d6" } [profile.dev] panic = "abort" diff --git a/README.md b/README.md index 6e3acc97..1bac9c63 100644 --- a/README.md +++ b/README.md @@ -25,29 +25,9 @@ [s2.dev](https://s2.dev) is a serverless datastore for real-time, streaming data. This repository contains: -- **[s2-cli](cli/)** - Command-line interface for S2 -- **[s2-lite](lite/)** - Open source, self-hostable server implementation of the [S2 API](https://s2.dev/docs/api) -- **[s2-sdk](sdk/)** - Rust SDK for S2 - -## Development - -Use the nightly Cargo dependency commands so that the repository publication cooldown applies: - -```bash -cargo +nightly add -cargo +nightly update -cargo +nightly update -p -cargo +nightly remove -cargo +nightly generate-lockfile -``` - -Use `--locked` with normal build, check, test, run, document, fetch, and metadata commands. The simulator is temporarily exempt until its separate lockfile is regenerated. The pull request dependency check verifies every proposed lock-file change before Rust build jobs start. - -Install the repository Cargo tools from Homebrew bottles: - -```bash -brew install cargo-deny cargo-nextest -``` +- **[s2-cli](cli/)** - The official S2 command-line interface +- **[s2-lite](lite/)** - An open source, self-hostable server implementation of the [S2 API](https://s2.dev/docs/api) +- **[s2-sdk](sdk/)** - The official Rust SDK for S2 ## Installation @@ -194,34 +174,6 @@ nc starwars.s2.dev 23 | s2 append s2://liteness/starwars Deploy `s2-lite` to Kubernetes using Helm. See the [Helm chart documentation](charts/s2-lite-helm/README.md) for installation instructions and configuration options. -### Storage - -Lite persists to an S3-compatible bucket or a local directory, or runs in-memory when neither is given. -The write-ahead log (WAL) shares the main store by default; it can be placed in a separate bucket or -directory to isolate WAL latency from flushes and compaction. `--path` applies to both stores. - -| Setting | Main store | WAL store | -| --- | --- | --- | -| S3 bucket | `--bucket` | `--wal-bucket` / `S2LITE_WAL_BUCKET` | -| Local directory | `--local-root` | `--wal-local-root` / `S2LITE_WAL_LOCAL_ROOT` | -| S3 endpoint | `AWS_ENDPOINT_URL_S3` | `S2LITE_WAL_AWS_ENDPOINT_URL_S3` | -| AWS region | `AWS_REGION` | `S2LITE_WAL_AWS_REGION` | -| Static credentials | `AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY` | `S2LITE_WAL_AWS_ACCESS_KEY_ID` + `S2LITE_WAL_AWS_SECRET_ACCESS_KEY` | -| Session token | `AWS_SESSION_TOKEN` | `S2LITE_WAL_AWS_SESSION_TOKEN` | - -Without static credentials, the standard AWS credential chain (profile, instance role, etc.) is used. -The WAL bucket inherits the main store's S3 settings; each `S2LITE_WAL_AWS_*` variable overrides just -that setting, except that a WAL key pair replaces the main credentials (and session token) as a set. -A WAL store requires a persistent main store. - -```bash -# LSM in S3, WAL on local disk -s2 lite --bucket my-bucket --wal-local-root /data/wal - -# LSM and WAL in separate buckets -s2 lite --bucket my-bucket --wal-bucket my-wal-bucket -``` - ### Monitoring `/health` will return 200 on success for readiness and liveness checks @@ -237,7 +189,7 @@ s2 lite --bucket my-bucket --wal-bucket my-wal-bucket Use `SL8_` prefixed environment variables, e.g.: ```bash -# Defaults to 50ms for S3, 5ms otherwise; follows the WAL store when set +# Defaults to 50ms for remote bucket / 5ms in-memory SL8_FLUSH_INTERVAL=10ms ``` diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index 55052994..4c989e96 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -2,51 +2,6 @@ All notable changes to this project will be documented in this file. -## [0.32.0] - 2026-09-25 - -### Features - -- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) - -### Bug Fixes - -- Skip s2s response compression the client refused with q=0 ([#781](https://github.com/s2-streamstore/s2/issues/781)) - - - -## [0.31.5] - 2026-09-22 - -### Bug Fixes - -- Preserve SSE read budgets across reconnects ([#727](https://github.com/s2-streamstore/s2/issues/727)) - - - -## [0.31.4] - 2026-09-16 - -### Miscellaneous Tasks - -- Sync specs submodule ([#739](https://github.com/s2-streamstore/s2/issues/739)) - - - -## [0.31.3] - 2026-09-11 - -### Features - -- `s2-stream-config` header for auto-created streams ([#718](https://github.com/s2-streamstore/s2/issues/718)) - - - -## [0.31.2] - 2026-09-10 - -### Bug Fixes - -- Reject NUL bytes in stream names and access token IDs ([#728](https://github.com/s2-streamstore/s2/issues/728)) -- Switch JSON extractor from sonic-rs back to serde_json ([#729](https://github.com/s2-streamstore/s2/issues/729)) - - - ## [0.31.1] - 2026-08-13 ### Features diff --git a/api/Cargo.toml b/api/Cargo.toml index 35cd9cee..f27ce3ee 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-api" -version = "0.32.0" +version = "0.31.1" description = "API types for S2, the durable streams API" edition.workspace = true license.workspace = true @@ -17,7 +17,7 @@ include = [ ] [features] -axum = ["dep:axum", "s2-common/axum"] +axum = ["dep:axum", "dep:sonic-rs", "s2-common/axum"] utoipa = ["dep:utoipa"] codegen = ["dep:prost-build"] @@ -36,6 +36,7 @@ prost = { workspace = true } s2-common = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +sonic-rs = { workspace = true, optional = true } strum = { workspace = true, features = ["derive"] } thiserror = { workspace = true } time = { workspace = true, features = ["serde", "formatting", "parsing"] } diff --git a/api/specs b/api/specs index edaa1fbc..973e0d92 160000 --- a/api/specs +++ b/api/specs @@ -1 +1 @@ -Subproject commit edaa1fbcb2507362d6c680c66cdc88660e73e036 +Subproject commit 973e0d92166ee1386b71d4e7ade8bd0fc2aaf4b8 diff --git a/api/src/data.rs b/api/src/data.rs index 9e53a92f..07e8e918 100644 --- a/api/src/data.rs +++ b/api/src/data.rs @@ -96,25 +96,6 @@ pub struct S2FormatHeader { pub s2_format: Format, } -#[rustfmt::skip] -#[derive(Debug)] -#[cfg_attr(feature = "utoipa", derive(utoipa::IntoParams))] -#[cfg_attr(feature = "utoipa", into_params(parameter_in = Header))] -pub struct S2StreamConfigHeader { - /// JSON-encoded `StreamConfig` to apply if the stream is created on append or read. - /// Unset fields inherit the basin's default stream configuration. - /// Ignored if the stream already exists. - /// Compact JSON is preferred. - #[cfg_attr(feature = "utoipa", param( - required = false, - rename = "s2-stream-config", - content_type = "application/json", - value_type = crate::v1::config::StreamConfig, - example = json!({"retention_policy":{"age":3600},"delete_on_empty":{"min_age_secs":300}}), - ))] - pub s2_stream_config: String, -} - #[rustfmt::skip] #[derive(Debug)] #[cfg_attr(feature = "utoipa", derive(utoipa::IntoParams))] @@ -201,10 +182,10 @@ pub mod extract { } } - fn classify_json_error(err: serde_json::Error) -> JsonExtractionRejection { - use serde_json::error::Category; + fn classify_sonic_error(err: sonic_rs::Error) -> JsonExtractionRejection { + use sonic_rs::error::Category; match err.classify() { - Category::Data => JsonExtractionRejection::DataError { + Category::TypeUnmatched | Category::NotFound => JsonExtractionRejection::DataError { status: http::StatusCode::UNPROCESSABLE_ENTITY, message: err.to_string().into(), }, @@ -212,7 +193,7 @@ pub mod extract { status: http::StatusCode::INTERNAL_SERVER_ERROR, message: err.to_string().into(), }, - Category::Syntax | Category::Eof => JsonExtractionRejection::SyntaxError { + _ => JsonExtractionRejection::SyntaxError { status: http::StatusCode::BAD_REQUEST, message: err.to_string().into(), }, @@ -242,9 +223,9 @@ pub mod extract { message: e.body_text().into(), } })?; - serde_json::from_slice(&bytes) + sonic_rs::from_slice(&bytes) .map(Self) - .map_err(classify_json_error) + .map_err(classify_sonic_error) } } @@ -274,9 +255,9 @@ pub mod extract { if bytes.is_empty() { return Ok(None); } - serde_json::from_slice(&bytes) + sonic_rs::from_slice(&bytes) .map(|v| Some(Self(v))) - .map_err(classify_json_error) + .map_err(classify_sonic_error) } } @@ -342,12 +323,15 @@ pub mod extract { stream::{AppendInput, AppendRecord, Header}, }; - fn parse_json(json: &[u8]) -> Result { - serde_json::from_slice(json).map_err(classify_json_error) + fn classify_json_error( + json: &[u8], + ) -> Result { + sonic_rs::from_slice(json).map_err(classify_sonic_error) } /// Verify that our rejection wrapper preserves axum's status code - /// classification for a variety of invalid JSON payloads. + /// classification for a variety of invalid JSON payloads, now using + /// sonic-rs as the deserializer. #[test] fn json_error_classification() { let cases: &[(&[u8], http::StatusCode)] = &[ @@ -371,7 +355,7 @@ pub mod extract { ]; for (input, expected_status) in cases { - let err = parse_json::(input).expect_err(&format!( + let err = classify_json_error::(input).expect_err(&format!( "expected error for {:?}", String::from_utf8_lossy(input) )); @@ -389,59 +373,24 @@ pub mod extract { #[test] fn valid_json_parses_successfully() { let input = br#"{"records": [], "match_seq_num": null}"#; - let result = parse_json::(input); + let result = classify_json_error::(input); assert!(result.is_ok()); } - /// A deeply nested value must never overflow the stack, wherever it - /// appears in the document: a wrong-typed value is rejected before it - /// is descended into, an unknown field is skipped iteratively, and - /// nesting that is actually deserialized hits the recursion limit. + /// Differential test: serialize with serde_json, deserialize with + /// both serde_json and sonic_rs, assert semantic equality. #[test] - fn deeply_nested_json_does_not_overflow_stack() { - const DEPTH: usize = 50_000; - let nested = format!("{}{}", "[".repeat(DEPTH), "]".repeat(DEPTH)); - let cases = [ - ( - format!(r#"{{"records":[{{"body":{nested}}}]}}"#), - Some(http::StatusCode::UNPROCESSABLE_ENTITY), - ), - (format!(r#"{{"records":[],"unknown":{nested}}}"#), None), - (nested.clone(), Some(http::StatusCode::UNPROCESSABLE_ENTITY)), - ]; - // Tokio's default worker stack size; unbounded recursion over - // 50k levels overflows it. - std::thread::Builder::new() - .stack_size(2 * 1024 * 1024) - .spawn(move || { - for (input, expected_status) in &cases { - let status = parse_json::(input.as_bytes()) - .err() - .map(|e| e.status()); - assert_eq!(status, *expected_status); - } - let err = parse_json::(nested.as_bytes()).unwrap_err(); - assert_eq!(err.status(), http::StatusCode::BAD_REQUEST); - assert!(err.body_text().contains("recursion limit exceeded")); - }) - .unwrap() - .join() - .unwrap(); - } - - /// Serialize with serde_json and deserialize again, asserting semantic - /// equality for shapes with custom (de)serialization. - #[test] - fn serde_json_roundtrip() { + fn serde_json_sonic_rs_roundtrip() { fn assert_roundtrip(input: &T) where T: serde::Serialize + serde::de::DeserializeOwned + std::fmt::Debug, { let json = serde_json::to_vec(input).unwrap(); - let parsed: T = parse_json(&json).unwrap(); + let from_serde: T = serde_json::from_slice(&json).unwrap(); + let from_sonic: T = sonic_rs::from_slice(&json).unwrap(); assert_eq!( - format!("{input:?}"), - format!("{parsed:?}"), + format!("{from_serde:?}"), + format!("{from_sonic:?}"), "roundtrip mismatch for {}", String::from_utf8_lossy(&json), ); @@ -466,7 +415,7 @@ pub mod extract { // StreamReconfiguration: exercises Maybe in all three states use s2_common::maybe::Maybe; - use crate::v1::config::{TimestampingMode, TimestampingReconfiguration}; + use crate::v1::config::{StorageClass, TimestampingMode, TimestampingReconfiguration}; // All fields unspecified (empty JSON object) assert_roundtrip(&StreamReconfiguration { @@ -477,7 +426,7 @@ pub mod extract { }); // Mix of specified-null and specified-value assert_roundtrip(&StreamReconfiguration { - storage_class: Maybe::Specified(Some("express".into())), + storage_class: Maybe::Specified(Some(StorageClass::Express)), retention_policy: Maybe::Specified(None), timestamping: Maybe::Specified(Some(TimestampingReconfiguration { mode: Maybe::Specified(Some(TimestampingMode::ClientRequire)), diff --git a/api/src/v1/access.rs b/api/src/v1/access.rs index a5dc1cc1..1dbc4173 100644 --- a/api/src/v1/access.rs +++ b/api/src/v1/access.rs @@ -193,8 +193,7 @@ impl From for AccessTokenInfo { #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] pub struct IssueAccessTokenRequest { /// Access token ID. - /// It must be unique to the account and between 1 and 96 bytes in length, and must not - /// contain NUL bytes. + /// It must be unique to the account and between 1 and 96 bytes in length. pub id: AccessTokenId, /// Expiration time in RFC 3339 format. /// If not set, the expiration will be set to that of the requestor's token. @@ -408,11 +407,9 @@ impl From for ReadWritePermissions { #[cfg_attr(feature = "utoipa", into_params(parameter_in = Query))] pub struct ListAccessTokensRequest { /// Filter to access tokens whose IDs begin with this prefix. - /// It must not contain NUL bytes. #[cfg_attr(feature = "utoipa", param(value_type = String, default = "", required = false))] pub prefix: Option, /// Filter to access tokens whose IDs lexicographically start after this string. - /// It must not contain NUL bytes. #[cfg_attr(feature = "utoipa", param(value_type = String, default = "", required = false))] pub start_after: Option, /// Number of results, up to a maximum of 1000. diff --git a/api/src/v1/config.rs b/api/src/v1/config.rs index 671c0cb8..fc523439 100644 --- a/api/src/v1/config.rs +++ b/api/src/v1/config.rs @@ -1,10 +1,37 @@ -use std::{str::FromStr, time::Duration}; +use std::time::Duration; -use compact_str::CompactString; -use http::{HeaderName, HeaderValue}; -use s2_common::{http::ParseableHeader, maybe::Maybe}; +use s2_common::maybe::Maybe; use serde::{Deserialize, Serialize}; +#[rustfmt::skip] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] +#[serde(rename_all = "kebab-case")] +pub enum StorageClass { + /// Append tail latency under 400 milliseconds with s2.dev. + Standard, + /// Append tail latency under 40 milliseconds with s2.dev. + Express, +} + +impl From for s2_common::config::StorageClass { + fn from(value: StorageClass) -> Self { + match value { + StorageClass::Express => Self::Express, + StorageClass::Standard => Self::Standard, + } + } +} + +impl From for StorageClass { + fn from(value: s2_common::config::StorageClass) -> Self { + match value { + s2_common::config::StorageClass::Express => Self::Express, + s2_common::config::StorageClass::Standard => Self::Standard, + } + } +} + #[rustfmt::skip] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] @@ -271,9 +298,8 @@ impl From for EncryptionAlgorithm { #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] pub struct StreamConfig { - /// [Storage class](https://s2.dev/docs/storage-classes) for recent writes. - #[cfg_attr(feature = "utoipa", schema(value_type = Option))] - pub storage_class: Option, + /// Storage class for recent writes. + pub storage_class: Option, /// Retention policy for the stream. /// If unspecified, the default is to retain records for 7 days. pub retention_policy: Option, @@ -294,7 +320,7 @@ impl StreamConfig { } = config; let config = StreamConfig { - storage_class, + storage_class: storage_class.map(Into::into), retention_policy: retention_policy.map(Into::into), timestamping: TimestampingConfig::to_opt(timestamping), delete_on_empty: DeleteOnEmptyConfig::to_opt(delete_on_empty), @@ -305,12 +331,6 @@ impl StreamConfig { Some(config) } } - - /// Encode as compact JSON for the `s2-stream-config` header. - pub fn to_header_value(&self) -> HeaderValue { - let json = serde_json::to_string(self).expect("StreamConfig serializes to JSON"); - HeaderValue::from_str(&json).expect("compact JSON of StreamConfig is a valid header value") - } } impl From for StreamConfig { @@ -323,7 +343,7 @@ impl From for StreamConfig { } = value; Self { - storage_class, + storage_class: Some(storage_class.into()), retention_policy: Some(retention_policy.into()), timestamping: Some(timestamping.into()), delete_on_empty: Some(delete_on_empty.into()), @@ -331,30 +351,6 @@ impl From for StreamConfig { } } -pub static STREAM_CONFIG_HEADER: HeaderName = HeaderName::from_static("s2-stream-config"); - -/// Value of the `s2-stream-config` header: a JSON-encoded [`StreamConfig`] to apply over the -/// basin's default stream config if the stream is created on append or read. Ignored if the -/// stream already exists. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct StreamConfigHeader(pub s2_common::config::OptionalStreamConfig); - -impl FromStr for StreamConfigHeader { - type Err = s2_common::ValidationError; - - fn from_str(s: &str) -> Result { - let config: StreamConfig = - serde_json::from_str(s).map_err(|e| format!("invalid JSON: {e}"))?; - Ok(Self(config.try_into()?)) - } -} - -impl ParseableHeader for StreamConfigHeader { - fn name() -> &'static HeaderName { - &STREAM_CONFIG_HEADER - } -} - impl TryFrom for s2_common::config::OptionalStreamConfig { type Error = s2_common::ValidationError; @@ -372,7 +368,7 @@ impl TryFrom for s2_common::config::OptionalStreamConfig { }; let config = Self { - storage_class, + storage_class: storage_class.map(Into::into), retention_policy, timestamping: timestamping.map(Into::into).unwrap_or_default(), delete_on_empty: delete_on_empty.map(Into::into).unwrap_or_default(), @@ -386,10 +382,10 @@ impl TryFrom for s2_common::config::OptionalStreamConfig { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] pub struct StreamReconfiguration { - /// [Storage class](https://s2.dev/docs/storage-classes) for recent writes. + /// Storage class for recent writes. #[serde(default, skip_serializing_if = "Maybe::is_unspecified")] - #[cfg_attr(feature = "utoipa", schema(value_type = Option))] - pub storage_class: Maybe>, + #[cfg_attr(feature = "utoipa", schema(value_type = Option))] + pub storage_class: Maybe>, /// Retention policy for the stream. /// If unspecified, the default is to retain records for 7 days. #[serde(default, skip_serializing_if = "Maybe::is_unspecified")] @@ -417,7 +413,7 @@ impl TryFrom for s2_common::config::StreamReconfiguration } = value; Ok(Self { - storage_class, + storage_class: storage_class.map_opt(Into::into), retention_policy: retention_policy.try_map_opt(TryInto::try_into)?, timestamping: timestamping.map_opt(Into::into), delete_on_empty: delete_on_empty.map_opt(Into::into), @@ -435,7 +431,7 @@ impl From for StreamReconfiguration { } = value; Self { - storage_class, + storage_class: storage_class.map_opt(Into::into), retention_policy: retention_policy.map_opt(Into::into), timestamping: timestamping.map_opt(Into::into), delete_on_empty: delete_on_empty.map_opt(Into::into), @@ -570,8 +566,8 @@ mod tests { use super::*; - fn gen_storage_class() -> impl Strategy { - "[a-z][a-z0-9-]{0,30}".prop_map(CompactString::from) + fn gen_storage_class() -> impl Strategy { + prop_oneof![Just(StorageClass::Standard), Just(StorageClass::Express)] } fn gen_timestamping_mode() -> impl Strategy { @@ -728,7 +724,7 @@ mod tests { ) .prop_map(|(sc, rp, ts_mode, ts_uncapped, doe)| { s2_common::config::OptionalStreamConfig { - storage_class: sc, + storage_class: sc.map(Into::into), retention_policy: rp.map(|rp| match rp { RetentionPolicy::Age(secs) => { s2_common::config::RetentionPolicy::Age(Duration::from_secs(secs)) @@ -800,7 +796,7 @@ mod tests { prop_assert_eq!( merged.storage_class, - stream.storage_class.or(basin.storage_class) + stream.storage_class.or(basin.storage_class).unwrap_or_default() ); prop_assert_eq!( merged.retention_policy, @@ -856,7 +852,7 @@ mod tests { new_rp_secs in 1u64..u64::MAX, ) { let reconfig = s2_common::config::StreamReconfiguration { - storage_class: Maybe::Specified(Some(new_sc.clone())), + storage_class: Maybe::Specified(Some(new_sc.into())), retention_policy: Maybe::Specified(Some( s2_common::config::RetentionPolicy::Age(Duration::from_secs(new_rp_secs)) )), @@ -864,7 +860,7 @@ mod tests { }; let result = base.reconfigure(reconfig); - prop_assert_eq!(result.storage_class, Some(new_sc)); + prop_assert_eq!(result.storage_class, Some(new_sc.into())); prop_assert_eq!( result.retention_policy, Some(s2_common::config::RetentionPolicy::Age(Duration::from_secs(new_rp_secs))) @@ -879,7 +875,7 @@ mod tests { ) { // non-default storage class -> Some let internal = s2_common::config::OptionalStreamConfig { - storage_class: Some(sc), + storage_class: Some(sc.into()), ..Default::default() }; prop_assert!(StreamConfig::to_opt(internal).is_some()); @@ -927,7 +923,7 @@ mod tests { ) { let base = s2_common::config::BasinConfig { default_stream_config: s2_common::config::OptionalStreamConfig { - storage_class: base_sc, + storage_class: base_sc.map(Into::into), ..Default::default() }, stream_cipher: base_algorithm.map(Into::into), @@ -958,7 +954,7 @@ mod tests { let reconfig = s2_common::config::BasinReconfiguration { default_stream_config: Maybe::Specified(Some(s2_common::config::StreamReconfiguration { - storage_class: Maybe::Specified(Some(new_sc.clone())), + storage_class: Maybe::Specified(Some(new_sc.into())), ..Default::default() })), stream_cipher: Maybe::Specified(Some(new_algorithm.into())), @@ -967,7 +963,7 @@ mod tests { }; let result = base.reconfigure(reconfig); - prop_assert_eq!(result.default_stream_config.storage_class, Some(new_sc)); + prop_assert_eq!(result.default_stream_config.storage_class, Some(new_sc.into())); prop_assert_eq!(result.stream_cipher, Some(new_algorithm.into())); prop_assert_eq!(result.create_stream_on_append, new_on_append); } @@ -1061,67 +1057,4 @@ mod tests { "delete_on_empty.min_age should be None" ); } - - #[test] - fn stream_config_header_parses_and_validates() { - let header: StreamConfigHeader = - r#"{"retention_policy":{"age":3600},"delete_on_empty":{"min_age_secs":300}}"# - .parse() - .unwrap(); - assert_eq!( - header.0, - s2_common::config::OptionalStreamConfig { - retention_policy: Some(s2_common::config::RetentionPolicy::Age( - Duration::from_secs(3600) - )), - delete_on_empty: s2_common::config::OptionalDeleteOnEmptyConfig { - min_age: Some(Duration::from_secs(300)), - }, - ..Default::default() - } - ); - - for spaced in [ - r#"{ "retention_policy": { "age": 3600 }, "delete_on_empty": { "min_age_secs": 300 } }"#, - "{\t\"delete_on_empty\":\t{\"min_age_secs\":\t300},\t\"retention_policy\":\t{\"age\":\t3600}\t}", - " {\"retention_policy\":{\"age\":3600},\"delete_on_empty\":{\"min_age_secs\":300}} ", - ] { - let parsed: StreamConfigHeader = spaced.parse().unwrap(); - assert_eq!(parsed, header, "{spaced:?}"); - } - - let empty: StreamConfigHeader = "{}".parse().unwrap(); - assert_eq!(empty.0, Default::default()); - - let invalid_json = "not json".parse::().unwrap_err(); - assert!(invalid_json.to_string().contains("invalid JSON")); - - let invalid_age = - r#"{"retention_policy":{"age":0}}"#.parse::().unwrap_err(); - assert!( - invalid_age - .to_string() - .contains("age must be greater than 0 seconds"), - "{invalid_age}" - ); - } - - #[test] - fn stream_config_header_value_roundtrips() { - let config = StreamConfig { - storage_class: Some("express".into()), - retention_policy: Some(RetentionPolicy::Infinite(InfiniteRetention {})), - timestamping: Some(TimestampingConfig { - mode: Some(TimestampingMode::ClientRequire), - uncapped: Some(true), - }), - delete_on_empty: Some(DeleteOnEmptyConfig { min_age_secs: 60 }), - }; - let value = config.to_header_value(); - let parsed: StreamConfigHeader = value.to_str().unwrap().parse().unwrap(); - assert_eq!( - parsed.0, - s2_common::config::OptionalStreamConfig::try_from(config).unwrap() - ); - } } diff --git a/api/src/v1/location.rs b/api/src/v1/location.rs index aacb78a4..408486e7 100644 --- a/api/src/v1/location.rs +++ b/api/src/v1/location.rs @@ -1,4 +1,3 @@ -use compact_str::CompactString; use s2_common::{self, location::LocationName}; use serde::{Deserialize, Serialize}; @@ -10,31 +9,13 @@ pub struct LocationInfo { pub name: LocationName, /// Location represents a private placement, limited by account. pub is_private: bool, - /// [Storage classes](https://s2.dev/docs/storage-classes) available to the account in this location. - #[serde(default, skip_serializing_if = "Option::is_none")] - #[cfg_attr(feature = "utoipa", schema(value_type = Option>))] - pub storage_classes: Option>, - /// Default [storage class](https://s2.dev/docs/storage-classes) for this location. - #[serde(default, skip_serializing_if = "Option::is_none")] - #[cfg_attr(feature = "utoipa", schema(value_type = Option))] - pub default_storage_class: Option, } impl From for LocationInfo { fn from(value: s2_common::location::LocationInfo) -> Self { - let s2_common::location::LocationInfo { - name, - is_private, - storage_classes, - default_storage_class, - } = value; + let s2_common::location::LocationInfo { name, is_private } = value; - Self { - name, - is_private, - storage_classes: Some(storage_classes), - default_storage_class: Some(default_storage_class), - } + Self { name, is_private } } } diff --git a/api/src/v1/stream/extract.rs b/api/src/v1/stream/extract.rs index deae94fb..6d508d45 100644 --- a/api/src/v1/stream/extract.rs +++ b/api/src/v1/stream/extract.rs @@ -17,7 +17,7 @@ use crate::{ extract::{JsonExtractionRejection, ProtoRejection}, }, mime::JsonOrProto, - v1::{config::StreamConfigHeader, stream::sse::LastEventId}, + v1::stream::sse::LastEventId, }; #[derive(Debug, thiserror::Error)] @@ -54,9 +54,6 @@ where async fn from_request(req: Request, state: &S) -> Result { let content_type = crate::mime::content_type(req.headers()); let encryption_key = parse_header_opt::(req.headers())?; - let create_stream_config_patch = parse_header_opt::(req.headers())? - .map(|header| header.0) - .unwrap_or_default(); if content_type.as_ref().is_some_and(crate::mime::is_s2s_proto) { let response_compression = @@ -91,7 +88,6 @@ where return Ok(Self::S2s { encryption_key, - create_stream_config_patch, inputs: Box::pin(inputs), response_compression, }); @@ -121,7 +117,6 @@ where Ok(Self::Unary { encryption_key, - create_stream_config_patch, input, response_mime, }) @@ -137,16 +132,12 @@ where async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { let content_type = crate::mime::content_type(&parts.headers); let encryption_key = parse_header_opt::(&parts.headers)?; - let create_stream_config_patch = parse_header_opt::(&parts.headers)? - .map(|header| header.0) - .unwrap_or_default(); if content_type.as_ref().is_some_and(crate::mime::is_s2s_proto) { let response_compression = s2s::CompressionAlgorithm::from_accept_encoding(&parts.headers); return Ok(Self::S2s { encryption_key, - create_stream_config_patch, response_compression, }); } @@ -159,7 +150,6 @@ where let last_event_id = parse_header_opt::(&parts.headers)?; return Ok(Self::EventStream { encryption_key, - create_stream_config_patch, format, last_event_id, }); @@ -172,7 +162,6 @@ where Ok(Self::Unary { encryption_key, - create_stream_config_patch, format, response_mime, }) diff --git a/api/src/v1/stream/mod.rs b/api/src/v1/stream/mod.rs index 2d8d66e8..730820da 100644 --- a/api/src/v1/stream/mod.rs +++ b/api/src/v1/stream/mod.rs @@ -11,7 +11,6 @@ use std::time::Duration; use futures_core::stream::BoxStream; use itertools::Itertools as _; use s2_common::{ - config::OptionalStreamConfig, encryption::EncryptionKey, record, stream::{StreamName, StreamNamePrefix, StreamNameStartAfter}, @@ -55,11 +54,9 @@ impl From for StreamInfo { #[cfg_attr(feature = "utoipa", into_params(parameter_in = Query))] pub struct ListStreamsRequest { /// Filter to streams whose names begin with this prefix. - /// It must not contain NUL bytes. #[cfg_attr(feature = "utoipa", param(value_type = String, default = "", required = false))] pub prefix: Option, /// Filter to streams whose names lexicographically start after this string. - /// It must not contain NUL bytes. #[cfg_attr(feature = "utoipa", param(value_type = String, default = "", required = false))] pub start_after: Option, /// Number of results, up to a maximum of 1000. @@ -85,7 +82,7 @@ pub struct ListStreamsResponse { #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] pub struct CreateStreamRequest { /// Stream name that is unique to the basin. - /// It can be between 1 and 512 bytes in length, and must not contain NUL bytes. + /// It can be between 1 and 512 bytes in length. pub stream: StreamName, /// Stream configuration. pub config: Option, @@ -211,24 +208,18 @@ pub enum ReadRequest { /// Unary Unary { encryption_key: Option, - /// Parsed `s2-stream-config` header; empty if absent. - create_stream_config_patch: OptionalStreamConfig, format: Format, response_mime: JsonOrProto, }, /// Server-Sent Events streaming response EventStream { encryption_key: Option, - /// Parsed `s2-stream-config` header; empty if absent. - create_stream_config_patch: OptionalStreamConfig, format: Format, last_event_id: Option, }, /// S2S streaming response S2s { encryption_key: Option, - /// Parsed `s2-stream-config` header; empty if absent. - create_stream_config_patch: OptionalStreamConfig, response_compression: s2s::CompressionAlgorithm, }, } @@ -237,16 +228,12 @@ pub enum AppendRequest { /// Unary Unary { encryption_key: Option, - /// Parsed `s2-stream-config` header; empty if absent. - create_stream_config_patch: OptionalStreamConfig, input: s2_common::stream::AppendInput, response_mime: JsonOrProto, }, /// S2S bi-directional streaming S2s { encryption_key: Option, - /// Parsed `s2-stream-config` header; empty if absent. - create_stream_config_patch: OptionalStreamConfig, inputs: BoxStream<'static, Result>, response_compression: s2s::CompressionAlgorithm, }, @@ -257,25 +244,21 @@ impl std::fmt::Debug for AppendRequest { match self { AppendRequest::Unary { encryption_key, - create_stream_config_patch, input, response_mime: response, } => f .debug_struct("AppendRequest::Unary") .field("encryption_key", encryption_key) - .field("create_stream_config_patch", create_stream_config_patch) .field("input", input) .field("response", response) .finish(), AppendRequest::S2s { encryption_key, - create_stream_config_patch, response_compression, .. } => f .debug_struct("AppendRequest::S2s") .field("encryption_key", encryption_key) - .field("create_stream_config_patch", create_stream_config_patch) .field("response_compression", response_compression) .finish(), } diff --git a/api/src/v1/stream/s2s.rs b/api/src/v1/stream/s2s.rs index ae6c6304..517745d3 100644 --- a/api/src/v1/stream/s2s.rs +++ b/api/src/v1/stream/s2s.rs @@ -71,11 +71,7 @@ impl CompressionAlgorithm { for header_value in headers.get_all(http::header::ACCEPT_ENCODING) { if let Ok(value) = header_value.to_str() { for encoding in value.split(',') { - let mut parts = encoding.split(';'); - let encoding = parts.next().unwrap_or("").trim(); - if parts.any(is_zero_qvalue) { - continue; - } + let encoding = encoding.trim().split(';').next().unwrap_or("").trim(); if encoding.eq_ignore_ascii_case("zstd") { return Self::Zstd; } else if encoding.eq_ignore_ascii_case("gzip") { @@ -88,15 +84,6 @@ impl CompressionAlgorithm { } } -/// Whether an `Accept-Encoding` parameter is a `q=0` weight, which marks the coding as -/// "not acceptable" (RFC 9110 §12.4.2). -fn is_zero_qvalue(param: &str) -> bool { - let Some((name, value)) = param.split_once('=') else { - return false; - }; - name.trim().eq_ignore_ascii_case("q") && value.trim().parse::().is_ok_and(|q| q == 0.0) -} - #[derive(Debug, Clone, PartialEq, Eq)] pub struct CompressedData { compression: CompressionAlgorithm, @@ -591,26 +578,6 @@ mod test { assert_eq!(algo, CompressionAlgorithm::Gzip); } - #[rstest::rstest] - #[case("zstd;q=0, gzip", CompressionAlgorithm::Gzip)] - #[case("zstd; q=0.0, gzip;q=0.5", CompressionAlgorithm::Gzip)] - #[case("gzip;q=0", CompressionAlgorithm::None)] - #[case("gzip;Q=0.000, zstd;q=0", CompressionAlgorithm::None)] - #[case("zstd;q=0.001", CompressionAlgorithm::Zstd)] - fn from_accept_encoding_skips_refused_codings( - #[case] accept_encoding: &'static str, - #[case] expected: CompressionAlgorithm, - ) { - let mut headers = http::HeaderMap::new(); - headers.insert( - http::header::ACCEPT_ENCODING, - HeaderValue::from_static(accept_encoding), - ); - - let algo = CompressionAlgorithm::from_accept_encoding(&headers); - assert_eq!(algo, expected); - } - #[test] fn from_accept_encoding_defaults_to_none() { let headers = http::HeaderMap::new(); diff --git a/api/src/v1/stream/sse.rs b/api/src/v1/stream/sse.rs index 7fad5d0f..f6c5b955 100644 --- a/api/src/v1/stream/sse.rs +++ b/api/src/v1/stream/sse.rs @@ -9,11 +9,8 @@ static LAST_EVENT_ID_HEADER: http::HeaderName = http::HeaderName::from_static("l #[derive(Debug, Clone, Copy)] pub struct LastEventId { - /// Sequence number of the last delivered record. pub seq_num: u64, - /// Total records delivered. pub count: usize, - /// Total metered bytes delivered. pub bytes: usize, } diff --git a/charts/s2-lite-helm/Chart.yaml b/charts/s2-lite-helm/Chart.yaml index 953a1674..54765424 100644 --- a/charts/s2-lite-helm/Chart.yaml +++ b/charts/s2-lite-helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: s2-lite-helm description: Self-hostable S2 streaming datastore using SlateDB on object storage type: application -version: 0.1.71 -appVersion: "0.43.0" +version: 0.1.63 +appVersion: "0.42.7" keywords: - s2 - streaming diff --git a/charts/s2-lite-helm/README.md b/charts/s2-lite-helm/README.md index ae02ba60..e019e02e 100644 --- a/charts/s2-lite-helm/README.md +++ b/charts/s2-lite-helm/README.md @@ -54,40 +54,6 @@ helm install my-s2-lite s2/s2-lite-helm \ Supports AWS S3, MinIO, Tigris, Cloudflare R2, and other S3-compatible services. -### Persistent volume - -```bash -helm install my-s2-lite s2/s2-lite-helm \ - --set persistentVolume.enabled=true \ - --set persistentVolume.size=20Gi -``` - -Stores the database on a `PersistentVolumeClaim` instead of a bucket. Set -`persistentVolume.existingClaim` to use your own claim. - -### Separate WAL storage - -The write-ahead log can live in a separate bucket or on its own persistent -volume (see [Storage](../../README.md#storage)). Requires `objectStorage` or -`persistentVolume`. - -```bash -# WAL on a persistent volume -helm install my-s2-lite s2/s2-lite-helm \ - --set objectStorage.enabled=true \ - --set objectStorage.bucket=my-bucket \ - --set walStorage.persistentVolume.enabled=true - -# WAL in a separate bucket -helm install my-s2-lite s2/s2-lite-helm \ - --set objectStorage.enabled=true \ - --set objectStorage.bucket=my-bucket \ - --set walStorage.bucket=my-wal-bucket -``` - -Keep the WAL location the same across upgrades; changing it does not migrate -WAL data and the server starts without an error. - ### TLS Configuration **Self-signed certificate (for dev/testing):** @@ -138,11 +104,6 @@ Common configurations: | `objectStorage.enabled` | Enable S3-compatible storage | `false` | | `objectStorage.bucket` | S3 bucket name | `""` | | `objectStorage.path` | Path prefix within bucket | `""` | -| `persistentVolume.enabled` | Store the database on a persistent volume | `false` | -| `persistentVolume.size` | Volume size | `10Gi` | -| `walStorage.bucket` | Separate S3 bucket for the WAL | `""` | -| `walStorage.persistentVolume.enabled` | Store the WAL on a persistent volume | `false` | -| `walStorage.persistentVolume.size` | WAL volume size | `10Gi` | | `metrics.serviceMonitor.enabled` | Enable Prometheus ServiceMonitor | `false` | ## Examples @@ -164,21 +125,6 @@ serviceAccount: helm install my-s2-lite s2/s2-lite-helm -f values.yaml ``` -### S3 with a local WAL volume - -```yaml -# values.yaml -objectStorage: - enabled: true - bucket: my-s3-bucket - -walStorage: - persistentVolume: - enabled: true - size: 20Gi - storageClass: gp3 -``` - ### Behind AWS Network Load Balancer ```yaml @@ -219,14 +165,3 @@ helm upgrade my-s2-lite s2/s2-lite-helm --version 0.1.0 helm uninstall my-s2-lite ``` -This deletes any `PersistentVolumeClaim` the chart created, and with it the data on the -volume. To keep the claim, annotate it before uninstalling: - -```yaml -persistentVolume: - annotations: - helm.sh/resource-policy: keep -``` - -Claims supplied via `existingClaim` are never deleted by the chart. - diff --git a/charts/s2-lite-helm/templates/NOTES.txt b/charts/s2-lite-helm/templates/NOTES.txt index ed9e7eb8..84b9755c 100644 --- a/charts/s2-lite-helm/templates/NOTES.txt +++ b/charts/s2-lite-helm/templates/NOTES.txt @@ -11,9 +11,7 @@ s2-lite is now running with the following configuration: - Replicas: {{ .Values.replicaCount }} - Image: {{ include "s2-lite.image" . }} -{{- $storageMode := include "s2-lite.storage.mode" . }} -{{- $walMode := include "s2-lite.walStorage.mode" . }} -{{- if eq $storageMode "bucket" }} +{{- if and .Values.objectStorage.enabled .Values.objectStorage.bucket }} - Storage: S3-compatible ({{ .Values.objectStorage.bucket }}) {{- if .Values.objectStorage.path }} - Path: {{ .Values.objectStorage.path }} @@ -21,16 +19,9 @@ s2-lite is now running with the following configuration: {{- if .Values.objectStorage.endpoint }} - Endpoint: {{ .Values.objectStorage.endpoint }} {{- end }} -{{- else if eq $storageMode "volume" }} - - Storage: Persistent volume ({{ include "s2-lite.claimName" (dict "root" . "name" "data" "volume" .Values.persistentVolume) }} at {{ .Values.persistentVolume.mountPath }}) {{- else }} - Storage: In-memory (data lost on pod restart) {{- end }} -{{- if eq $walMode "bucket" }} - - WAL: S3-compatible ({{ .Values.walStorage.bucket }}) -{{- else if eq $walMode "volume" }} - - WAL: Persistent volume ({{ include "s2-lite.claimName" (dict "root" . "name" "wal" "volume" .Values.walStorage.persistentVolume) }} at {{ .Values.walStorage.persistentVolume.mountPath }}) -{{- end }} Get the application URL by running these commands: export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "s2-lite.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") diff --git a/charts/s2-lite-helm/templates/_helpers.tpl b/charts/s2-lite-helm/templates/_helpers.tpl index faf28de6..62d8dd83 100644 --- a/charts/s2-lite-helm/templates/_helpers.tpl +++ b/charts/s2-lite-helm/templates/_helpers.tpl @@ -66,52 +66,3 @@ Create the image name {{- $tag := .Values.image.tag | default .Chart.AppVersion }} {{- printf "%s:%s" .Values.image.repository $tag }} {{- end }} - -{{/* -Main storage mode: "bucket", "volume", or "" for in-memory. -*/}} -{{- define "s2-lite.storage.mode" -}} -{{- if and .Values.objectStorage.enabled .Values.persistentVolume.enabled }} -{{- fail "objectStorage.enabled and persistentVolume.enabled are mutually exclusive" }} -{{- end }} -{{- if and .Values.objectStorage.enabled (not .Values.objectStorage.bucket) }} -{{- fail "objectStorage.enabled is true but objectStorage.bucket is not set" }} -{{- end }} -{{- if .Values.objectStorage.enabled }}bucket{{- else if .Values.persistentVolume.enabled }}volume{{- end }} -{{- end }} - -{{/* -WAL storage mode: "bucket", "volume", or "" when the WAL shares the main store. -*/}} -{{- define "s2-lite.walStorage.mode" -}} -{{- $bucket := .Values.walStorage.bucket }} -{{- $volume := .Values.walStorage.persistentVolume.enabled }} -{{- if and $bucket $volume }} -{{- fail "walStorage.bucket and walStorage.persistentVolume.enabled are mutually exclusive" }} -{{- end }} -{{- if and (or $bucket $volume) (not (include "s2-lite.storage.mode" .)) }} -{{- fail "walStorage requires objectStorage or persistentVolume" }} -{{- end }} -{{- if $bucket }}bucket{{- else if $volume }}volume{{- end }} -{{- end }} - -{{/* -Persistent volumes in use, keyed by volume name. Parse with fromYaml. -*/}} -{{- define "s2-lite.persistentVolumes" -}} -{{- $volumes := dict }} -{{- if eq (include "s2-lite.storage.mode" .) "volume" }} -{{- $_ := set $volumes "data" .Values.persistentVolume }} -{{- end }} -{{- if eq (include "s2-lite.walStorage.mode" .) "volume" }} -{{- $_ := set $volumes "wal" .Values.walStorage.persistentVolume }} -{{- end }} -{{- toYaml $volumes }} -{{- end }} - -{{/* -PersistentVolumeClaim name for a volume. Takes (dict "root" $ "name" "volume" ). -*/}} -{{- define "s2-lite.claimName" -}} -{{- .volume.existingClaim | default (printf "%s-%s" (include "s2-lite.fullname" .root) .name) }} -{{- end }} diff --git a/charts/s2-lite-helm/templates/deployment.yaml b/charts/s2-lite-helm/templates/deployment.yaml index 377dadaa..34bee45b 100644 --- a/charts/s2-lite-helm/templates/deployment.yaml +++ b/charts/s2-lite-helm/templates/deployment.yaml @@ -1,6 +1,3 @@ -{{- $storageMode := include "s2-lite.storage.mode" . }} -{{- $walMode := include "s2-lite.walStorage.mode" . }} -{{- $volumes := include "s2-lite.persistentVolumes" . | fromYaml }} apiVersion: apps/v1 kind: Deployment metadata: @@ -55,42 +52,27 @@ spec: {{- fail "tls.enabled is true but neither tls.cert/tls.key nor tls.selfSigned are configured" }} {{- end }} {{- end }} - {{- if eq $storageMode "bucket" }} + {{- if .Values.objectStorage.enabled }} + {{- if not .Values.objectStorage.bucket }} + {{- fail "objectStorage.enabled is true but objectStorage.bucket is not set" }} + {{- end }} - --bucket - {{ .Values.objectStorage.bucket | quote }} - {{- with .Values.objectStorage.path }} + {{- if .Values.objectStorage.path }} - --path - - {{ . | quote }} + - {{ .Values.objectStorage.path | quote }} {{- end }} - {{- else if eq $storageMode "volume" }} - - --local-root - - {{ .Values.persistentVolume.mountPath | quote }} - {{- end }} - {{- if eq $walMode "bucket" }} - - --wal-bucket - - {{ .Values.walStorage.bucket | quote }} - {{- else if eq $walMode "volume" }} - - --wal-local-root - - {{ .Values.walStorage.persistentVolume.mountPath | quote }} {{- end }} ports: - name: http containerPort: {{ .Values.service.targetPort }} protocol: TCP - {{- if or (and .Values.objectStorage.enabled .Values.objectStorage.endpoint) .Values.walStorage.endpoint .Values.walStorage.region .Values.env }} + {{- if or (and .Values.objectStorage.enabled .Values.objectStorage.endpoint) .Values.env }} env: {{- if and .Values.objectStorage.enabled .Values.objectStorage.endpoint }} - name: AWS_ENDPOINT_URL_S3 value: {{ .Values.objectStorage.endpoint | quote }} {{- end }} - {{- with .Values.walStorage.endpoint }} - - name: S2LITE_WAL_AWS_ENDPOINT_URL_S3 - value: {{ . | quote }} - {{- end }} - {{- with .Values.walStorage.region }} - - name: S2LITE_WAL_AWS_REGION - value: {{ . | quote }} - {{- end }} {{- with .Values.env }} {{- toYaml . | nindent 8 }} {{- end }} @@ -126,27 +108,14 @@ spec: {{- end }} resources: {{- toYaml .Values.resources | nindent 10 }} - {{- if or $volumes .Values.volumeMounts }} - volumeMounts: - {{- range $name, $volume := $volumes }} - - name: {{ $name }} - mountPath: {{ $volume.mountPath | quote }} - {{- end }} {{- with .Values.volumeMounts }} + volumeMounts: {{- toYaml . | nindent 8 }} {{- end }} - {{- end }} - {{- if or $volumes .Values.volumes }} - volumes: - {{- range $name, $volume := $volumes }} - - name: {{ $name }} - persistentVolumeClaim: - claimName: {{ include "s2-lite.claimName" (dict "root" $ "name" $name "volume" $volume) }} - {{- end }} {{- with .Values.volumes }} + volumes: {{- toYaml . | nindent 6 }} {{- end }} - {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/s2-lite-helm/templates/pvc.yaml b/charts/s2-lite-helm/templates/pvc.yaml deleted file mode 100644 index b19594c1..00000000 --- a/charts/s2-lite-helm/templates/pvc.yaml +++ /dev/null @@ -1,24 +0,0 @@ -{{- range $name, $volume := include "s2-lite.persistentVolumes" . | fromYaml }} -{{- if not $volume.existingClaim }} ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: {{ include "s2-lite.claimName" (dict "root" $ "name" $name "volume" $volume) }} - labels: - {{- include "s2-lite.labels" $ | nindent 4 }} - {{- with $volume.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - accessModes: - - ReadWriteOnce - {{- with $volume.storageClass }} - storageClassName: {{ . | quote }} - {{- end }} - resources: - requests: - storage: {{ $volume.size | quote }} -{{- end }} -{{- end }} diff --git a/charts/s2-lite-helm/values.yaml b/charts/s2-lite-helm/values.yaml index 22336d60..19087c99 100644 --- a/charts/s2-lite-helm/values.yaml +++ b/charts/s2-lite-helm/values.yaml @@ -102,8 +102,7 @@ ingress: # Storage Configuration # # By default, s2-lite runs in-memory (great for dev/testing). -# For persistent data, enable objectStorage with an S3-compatible bucket or -# persistentVolume for a local disk. Enable at most one. +# For persistent data, enable objectStorage with an S3-compatible bucket. # ------------------------------------------------------------------------------ objectStorage: @@ -119,45 +118,6 @@ objectStorage: # R2: "https://.r2.cloudflarestorage.com" endpoint: "" -# Persistent volume for the database (--local-root) -persistentVolume: - enabled: false - mountPath: /data - size: 10Gi - storageClass: "" - # Use an existing PersistentVolumeClaim instead of creating one - existingClaim: "" - # helm uninstall deletes the created claim; add helm.sh/resource-policy: keep to retain it - annotations: {} - -# ------------------------------------------------------------------------------ -# WAL Storage Configuration -# -# By default the write-ahead log (WAL) is stored with the database. Place it in -# a separate bucket or on its own persistent volume to isolate WAL latency from -# flushes and compaction. Requires objectStorage or persistentVolume; set at -# most one of bucket or persistentVolume.enabled. -# -# Keep the WAL location the same across upgrades: changing it does not migrate -# WAL data and the server starts without an error. -# ------------------------------------------------------------------------------ - -walStorage: - # S3 bucket for the WAL (--wal-bucket). Shares objectStorage's AWS - # configuration unless overridden below. For different credentials, set - # S2LITE_WAL_AWS_ACCESS_KEY_ID / S2LITE_WAL_AWS_SECRET_ACCESS_KEY via env. - bucket: "" - endpoint: "" - region: "" - # Persistent volume for the WAL (--wal-local-root) - persistentVolume: - enabled: false - mountPath: /wal - size: 10Gi - storageClass: "" - existingClaim: "" - annotations: {} - # ------------------------------------------------------------------------------ # Observability # ------------------------------------------------------------------------------ diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 9004262c..98eef49f 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -2,62 +2,6 @@ All notable changes to this project will be documented in this file. -## [0.43.0] - 2026-09-25 - -### Features - -- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) - -### Bug Fixes - -- Show match-none token scopes as no access, not as wildcards ([#782](https://github.com/s2-streamstore/s2/issues/782)) - - - -## [0.42.14] - 2026-09-24 - - - -## [0.42.13] - 2026-09-22 - -### Features - -- Support a separate WAL object store ([#766](https://github.com/s2-streamstore/s2/issues/766)) - - - -## [0.42.12] - 2026-09-16 - -### Miscellaneous Tasks - -- Update Cargo.lock dependencies - - - -## [0.42.11] - 2026-09-11 - -### Features - -- `s2-stream-config` header for auto-created streams ([#718](https://github.com/s2-streamstore/s2/issues/718)) - - - -## [0.42.10] - 2026-09-11 - - - -## [0.42.9] - 2026-09-10 - - - -## [0.42.8] - 2026-09-01 - -### Bug Fixes - -- Require http2 ([#710](https://github.com/s2-streamstore/s2/issues/710)) - - - ## [0.42.7] - 2026-08-18 ### Bug Fixes diff --git a/cli/Cargo.toml b/cli/Cargo.toml index b5284ce4..abbd4eac 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-cli" -version = "0.43.0" +version = "0.42.7" description = "CLI for S2" edition.workspace = true license.workspace = true @@ -21,7 +21,6 @@ bytes = { workspace = true } clap = { workspace = true, features = ["derive"] } color-print = { workspace = true } colored = { workspace = true } -compact_str = { workspace = true, features = ["serde"] } config = { workspace = true } dirs = { workspace = true } fs2 = { workspace = true } @@ -72,9 +71,6 @@ serde_json = { workspace = true } [dev-dependencies] assert_cmd = "2.2" -http-body-util = "0.1" -hyper = { version = "1", features = ["http2", "server"] } -hyper-util = { version = "0.1", features = ["http2", "server", "tokio"] } predicates = "3.1" proptest = { workspace = true } rstest = { workspace = true } diff --git a/cli/schema.json b/cli/schema.json index c785bd21..c6f9c997 100644 --- a/cli/schema.json +++ b/cli/schema.json @@ -87,10 +87,14 @@ "type": "object", "properties": { "storage_class": { - "description": "[Storage class](https://s2.dev/docs/storage-classes) for recent writes.", - "type": [ - "string", - "null" + "description": "Storage class for recent writes.", + "anyOf": [ + { + "$ref": "#/$defs/StorageClass" + }, + { + "type": "null" + } ], "default": null }, @@ -130,6 +134,14 @@ }, "additionalProperties": false }, + "StorageClass": { + "description": "Storage class for recent writes.", + "type": "string", + "enum": [ + "standard", + "express" + ] + }, "RetentionPolicy": { "description": "Retain records unless explicitly trimmed (\"infinite\"), or automatically trim records older than the given duration (e.g. \"7days\", \"1week\"). Age durations must be greater than 0 seconds.", "type": "string", diff --git a/cli/src/apply.rs b/cli/src/apply.rs index 372b2d0e..7235508d 100644 --- a/cli/src/apply.rs +++ b/cli/src/apply.rs @@ -3,19 +3,88 @@ use std::path::Path; use colored::Colorize; -use miette::IntoDiagnostic; use s2_common::{ basin::BasinName, - config::{RetentionPolicy, StreamConfig, TimestampingMode}, + config::{ + BasinConfig, OptionalDeleteOnEmptyConfig, OptionalStreamConfig, OptionalTimestampingConfig, + RetentionPolicy, StorageClass, StreamConfig, TimestampingMode, + }, encryption::EncryptionAlgorithm, stream::StreamName, }; -use s2_sdk::{ - error::{ErrorCode, RequestError, StatusCode}, - types::BasinConfig, -}; +use s2_sdk::error::{ErrorCode, RequestError}; -use crate::types::resolve_stream_config; +fn basin_config_from_sdk(config: s2_sdk::types::BasinConfig) -> BasinConfig { + BasinConfig { + default_stream_config: config + .default_stream_config + .map(optional_stream_config_from_sdk) + .unwrap_or_default(), + stream_cipher: config.stream_cipher, + create_stream_on_append: config.create_stream_on_append, + create_stream_on_read: config.create_stream_on_read, + } +} + +fn stream_config_from_sdk(config: s2_sdk::types::StreamConfig) -> StreamConfig { + optional_stream_config_from_sdk(config).into() +} + +fn optional_stream_config_from_sdk(config: s2_sdk::types::StreamConfig) -> OptionalStreamConfig { + OptionalStreamConfig { + storage_class: config.storage_class.map(storage_class_from_sdk), + retention_policy: config.retention_policy.map(retention_policy_from_sdk), + timestamping: config + .timestamping + .map(timestamping_from_sdk) + .unwrap_or_default(), + delete_on_empty: config + .delete_on_empty + .map(delete_on_empty_from_sdk) + .unwrap_or_default(), + } +} + +fn storage_class_from_sdk(storage_class: s2_sdk::types::StorageClass) -> StorageClass { + match storage_class { + s2_sdk::types::StorageClass::Standard => StorageClass::Standard, + s2_sdk::types::StorageClass::Express => StorageClass::Express, + } +} + +fn retention_policy_from_sdk(retention_policy: s2_sdk::types::RetentionPolicy) -> RetentionPolicy { + match retention_policy { + s2_sdk::types::RetentionPolicy::Age(secs) => { + RetentionPolicy::Age(std::time::Duration::from_secs(secs)) + } + s2_sdk::types::RetentionPolicy::Infinite => RetentionPolicy::Infinite(), + } +} + +fn timestamping_from_sdk( + timestamping: s2_sdk::types::TimestampingConfig, +) -> OptionalTimestampingConfig { + OptionalTimestampingConfig { + mode: timestamping.mode.map(timestamping_mode_from_sdk), + uncapped: timestamping.uncapped, + } +} + +fn timestamping_mode_from_sdk(mode: s2_sdk::types::TimestampingMode) -> TimestampingMode { + match mode { + s2_sdk::types::TimestampingMode::ClientPrefer => TimestampingMode::ClientPrefer, + s2_sdk::types::TimestampingMode::ClientRequire => TimestampingMode::ClientRequire, + s2_sdk::types::TimestampingMode::Arrival => TimestampingMode::Arrival, + } +} + +fn delete_on_empty_from_sdk( + delete_on_empty: s2_sdk::types::DeleteOnEmptyConfig, +) -> OptionalDeleteOnEmptyConfig { + OptionalDeleteOnEmptyConfig { + min_age: Some(std::time::Duration::from_secs(delete_on_empty.min_age_secs)), + } +} fn basin_config_to_sdk(config: s2_resource_spec::BasinConfig) -> s2_sdk::types::BasinConfig { let mut sdk_config = s2_sdk::types::BasinConfig::new(); @@ -38,7 +107,7 @@ fn basin_config_to_sdk(config: s2_resource_spec::BasinConfig) -> s2_sdk::types:: fn stream_config_to_sdk(config: s2_resource_spec::StreamConfig) -> s2_sdk::types::StreamConfig { let mut sdk_config = s2_sdk::types::StreamConfig::new(); if let Some(storage_class) = config.storage_class { - sdk_config = sdk_config.with_storage_class(storage_class); + sdk_config = sdk_config.with_storage_class(storage_class_to_sdk(storage_class)); } if let Some(retention_policy) = config.retention_policy { sdk_config = sdk_config.with_retention_policy(retention_policy_to_sdk(retention_policy)); @@ -52,6 +121,15 @@ fn stream_config_to_sdk(config: s2_resource_spec::StreamConfig) -> s2_sdk::types sdk_config } +fn storage_class_to_sdk( + storage_class: s2_resource_spec::StorageClass, +) -> s2_sdk::types::StorageClass { + match storage_class { + s2_resource_spec::StorageClass::Standard => s2_sdk::types::StorageClass::Standard, + s2_resource_spec::StorageClass::Express => s2_sdk::types::StorageClass::Express, + } +} + fn retention_policy_to_sdk( retention_policy: s2_resource_spec::RetentionPolicy, ) -> s2_sdk::types::RetentionPolicy { @@ -212,26 +290,10 @@ async fn apply_stream( enum ResourceAction { Create, - Ensure { - diffs: Vec, - storage_class_unresolved: bool, - }, + Ensure(Vec), Unchanged, } -impl ResourceAction { - fn from_diff(diffs: Vec, storage_class_unresolved: bool) -> Self { - if diffs.is_empty() && !storage_class_unresolved { - Self::Unchanged - } else { - Self::Ensure { - diffs, - storage_class_unresolved, - } - } - } -} - struct FieldDiff { field: &'static str, old: String, @@ -245,6 +307,13 @@ fn is_not_found_error(error: &RequestError) -> bool { .is_some_and(|code| matches!(code, ErrorCode::BasinNotFound | ErrorCode::StreamNotFound)) } +fn format_storage_class(sc: StorageClass) -> &'static str { + match sc { + StorageClass::Standard => "standard", + StorageClass::Express => "express", + } +} + fn format_retention_policy(rp: RetentionPolicy) -> String { match rp { RetentionPolicy::Age(age) => humantime::format_duration(age).to_string(), @@ -271,10 +340,7 @@ fn default_stream_config_field(field: &'static str) -> &'static str { } } -fn diff_basin_config( - existing: &BasinConfig, - desired: &BasinConfig, -) -> miette::Result> { +fn diff_basin_config(existing: &BasinConfig, desired: &BasinConfig) -> Vec { let mut diffs = Vec::new(); if existing.stream_cipher != desired.stream_cipher { @@ -309,24 +375,8 @@ fn diff_basin_config( }); } - let existing_dsc = resolve_stream_config( - existing - .default_stream_config - .clone() - .unwrap_or_default() - .into(), - Default::default(), - ) - .into_diagnostic()?; - let desired_dsc = resolve_stream_config( - desired - .default_stream_config - .clone() - .unwrap_or_default() - .into(), - Default::default(), - ) - .into_diagnostic()?; + let existing_dsc: StreamConfig = existing.default_stream_config.clone().into(); + let desired_dsc: StreamConfig = desired.default_stream_config.clone().into(); for sd in diff_stream_configs(&existing_dsc, &desired_dsc) { diffs.push(FieldDiff { field: default_stream_config_field(sd.field), @@ -335,23 +385,17 @@ fn diff_basin_config( }); } - Ok(diffs) + diffs } fn diff_stream_configs(existing: &StreamConfig, desired: &StreamConfig) -> Vec { let mut diffs = Vec::new(); - if let Some(storage_class) = &desired.storage_class - && existing.storage_class.as_ref() != Some(storage_class) - { + if existing.storage_class != desired.storage_class { diffs.push(FieldDiff { field: "storage_class", - old: existing - .storage_class - .as_deref() - .unwrap_or("unspecified") - .to_owned(), - new: storage_class.to_string(), + old: format_storage_class(existing.storage_class).to_string(), + new: format_storage_class(desired.storage_class).to_string(), }); } @@ -434,7 +478,7 @@ fn spec_stream_fields(spec: &s2_resource_spec::StreamConfig) -> Vec { fields.push(FieldDiff { field: "storage_class", old: String::new(), - new: sc.to_string(), + new: format_storage_class(sc.clone().into()).to_string(), }); } if let Some(ref rp) = spec.retention_policy { @@ -478,18 +522,11 @@ fn print_basin_result(basin: &str, action: &ResourceAction) { ResourceAction::Create => { println!("{}", format!("+ basin {basin}").green().bold()); } - ResourceAction::Ensure { - diffs, - storage_class_unresolved, - } => { - let marker = if diffs.is_empty() { "?" } else { "~" }; - println!("{}", format!("{marker} basin {basin}").yellow().bold()); + ResourceAction::Ensure(diffs) => { + println!("{}", format!("~ basin {basin}").yellow().bold()); for diff in diffs { println!(" {}: {} → {}", diff.field, diff.old.dimmed(), diff.new); } - if *storage_class_unresolved { - println!(" default_stream_config.storage_class: resolved at apply"); - } } ResourceAction::Unchanged => { println!("{}", format!("= basin {basin}").dimmed()); @@ -502,23 +539,11 @@ fn print_stream_result(basin: &str, stream: &str, action: &ResourceAction) { ResourceAction::Create => { println!("{}", format!(" + stream {basin}/{stream}").green().bold()); } - ResourceAction::Ensure { - diffs, - storage_class_unresolved, - } => { - let marker = if diffs.is_empty() { "?" } else { "~" }; - println!( - "{}", - format!(" {marker} stream {basin}/{stream}") - .yellow() - .bold() - ); + ResourceAction::Ensure(diffs) => { + println!("{}", format!(" ~ stream {basin}/{stream}").yellow().bold()); for diff in diffs { println!(" {}: {} → {}", diff.field, diff.old.dimmed(), diff.new); } - if *storage_class_unresolved { - println!(" storage_class: resolved at apply"); - } } ResourceAction::Unchanged => { println!("{}", format!(" = stream {basin}/{stream}").dimmed()); @@ -547,55 +572,24 @@ fn print_stream_create(basin: &str, stream: &str, spec: &Option miette::Result<()> { validate(&spec)?; - let needs_location_default = spec.basins.iter().any(|basin| { - basin - .config - .as_ref() - .and_then(|config| config.default_stream_config.as_ref()) - .and_then(|config| config.storage_class.as_ref()) - .is_none() - }); - let default_storage_class = if needs_location_default { - match s2.get_default_location().await { - Ok(location) => location.default_storage_class, - Err(error) - if error.server_error().is_some_and(|error| { - matches!( - error.known_code(), - Some(ErrorCode::NotImplemented | ErrorCode::PermissionDenied) - ) || error.status == StatusCode::NOT_FOUND - }) => - { - None - } - Err(error) => { - return Err(miette::miette!( - "failed to get default storage class: {error}" - )); - } - } - } else { - None - }; - for basin_spec in spec.basins { - let mut desired_basin_config = basin_spec + let desired_basin_config = basin_spec .config .clone() - .map(basin_config_to_sdk) + .map(BasinConfig::from) .unwrap_or_default(); - let desired_basin_defaults = desired_basin_config - .default_stream_config - .get_or_insert_default(); - if desired_basin_defaults.storage_class.is_none() { - desired_basin_defaults.storage_class = default_storage_class.clone(); - } - let desired_basin_defaults = desired_basin_defaults.clone(); + let desired_basin_default_stream_config = + desired_basin_config.default_stream_config.clone(); let basin_action = match s2.get_basin_config(basin_spec.name.clone()).await { Ok(existing) => { - let diffs = diff_basin_config(&existing, &desired_basin_config)?; - ResourceAction::from_diff(diffs, desired_basin_defaults.storage_class.is_none()) + let existing = basin_config_from_sdk(existing); + let diffs = diff_basin_config(&existing, &desired_basin_config); + if diffs.is_empty() { + ResourceAction::Unchanged + } else { + ResourceAction::Ensure(diffs) + } } Err(e) if is_not_found_error(&e) => ResourceAction::Create, Err(e) => { @@ -624,20 +618,19 @@ pub async fn dry_run(s2: &s2_sdk::S2, spec: s2_resource_spec::Resources) -> miet .await { Ok(existing) => { - let existing = resolve_stream_config(existing.into(), Default::default()) - .into_diagnostic()?; - let desired_stream_config = resolve_stream_config( - stream_spec - .config - .clone() - .map(stream_config_to_sdk) - .unwrap_or_default() - .into(), - desired_basin_defaults.clone().into(), - ) - .into_diagnostic()?; + let existing = stream_config_from_sdk(existing); + let desired_stream_config = stream_spec + .config + .clone() + .map(OptionalStreamConfig::from) + .unwrap_or_default() + .merge(desired_basin_default_stream_config.clone()); let diffs = diff_stream_configs(&existing, &desired_stream_config); - ResourceAction::from_diff(diffs, desired_stream_config.storage_class.is_none()) + if diffs.is_empty() { + ResourceAction::Unchanged + } else { + ResourceAction::Ensure(diffs) + } } Err(e) if is_not_found_error(&e) => ResourceAction::Create, Err(e) => { diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 30f2baf7..74981c1a 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -1,7 +1,6 @@ use std::{num::NonZeroU64, path::PathBuf, time::Duration}; use clap::{Args, Parser, Subcommand, ValueEnum, builder::styling}; -use compact_str::CompactString; use s2_sdk::types::{ AccessTokenId, AccessTokenIdPrefix, AccessTokenIdStartAfter, BasinName, BasinNamePrefix, BasinNameStartAfter, EncryptionAlgorithm, EncryptionKey, FencingToken, StreamName, @@ -15,7 +14,7 @@ use crate::{ }, types::{ BasinConfig, Interval, LocationName, Operation, PermittedOperationGroups, - S2BasinAndMaybeStreamUri, S2BasinAndStreamUri, S2BasinUri, StreamConfig, + S2BasinAndMaybeStreamUri, S2BasinAndStreamUri, S2BasinUri, StorageClass, StreamConfig, }, }; @@ -192,7 +191,6 @@ pub enum Command { /// `+` create /// `~` ensure /// `=` unchanged - /// `?` storage-class default resolved at apply /// /// For IDE validation/autocomplete, add `$schema` at the top of each spec file: /// {"$schema":"https://raw.githubusercontent.com/s2-streamstore/s2/main/cli/schema.json","basins":[]} @@ -695,15 +693,6 @@ pub struct AppendArgs { #[command(flatten)] pub encryption_key: EncryptionKeyArgs, - - /// Stream configuration to apply if the stream is created on append. - /// Unset fields inherit the basin's default stream configuration. - /// Ignored if the stream already exists. - #[command( - flatten, - next_help_heading = "Stream configuration (applied only if the stream is created on append)" - )] - pub stream_config: StreamConfig, } #[derive(Args, Debug, Clone, Default)] @@ -782,15 +771,6 @@ pub struct ReadArgs { #[command(flatten)] pub encryption_key: EncryptionKeyArgs, - - /// Stream configuration to apply if the stream is created on read. - /// Unset fields inherit the basin's default stream configuration. - /// Ignored if the stream already exists. - #[command( - flatten, - next_help_heading = "Stream configuration (applied only if the stream is created on read)" - )] - pub stream_config: StreamConfig, } #[derive(Args, Debug)] @@ -836,7 +816,6 @@ pub struct ApplyArgs { /// `+` create /// `~` ensure /// `=` unchanged - /// `?` storage-class default resolved at apply #[arg(long)] pub dry_run: bool, /// Print the JSON Schema for the spec file format to stdout. @@ -851,7 +830,7 @@ pub struct BenchArgs { /// Storage class for the test stream. Uses basin default if not specified. #[arg(short = 'c', long)] - pub storage_class: Option, + pub storage_class: Option, /// Total metered record size in bytes (includes headers and overhead). #[arg( diff --git a/cli/src/diff.rs b/cli/src/diff.rs index 6ea1ffe3..75f612a4 100644 --- a/cli/src/diff.rs +++ b/cli/src/diff.rs @@ -1,12 +1,16 @@ -//! Diffing over typed presentation views of basins, streams, and access tokens. +//! Diffing over typed presentation views of Basins, Streams, and Access Tokens. +//! +//! - Configs are fetched as API DTOs and converted into `s2_common` domain types via the existing +//! `TryFrom` conversions, which also resolve omitted fields to their effective defaults (e.g. an +//! unset `storage_class` becomes `express`). +//! +//! - Domain values are rendered into `*View` structs and serialized to JSON, which is diffed +//! field-by-field. Views own all presentation: compact exact durations (`"7d"`, `"1h"`), +//! wire-format enum names, canonically ordered operation sets. use colored::Colorize; -use compact_str::CompactString; use s2_api::v1::access::AccessTokenInfo as ApiAccessTokenInfo; -use s2_common::{ - access::{PermittedOperationGroups, ReadWritePermissions, ResourceSet}, - config::StreamConfig, -}; +use s2_common::access::{PermittedOperationGroups, ReadWritePermissions, ResourceSet}; use s2_sdk::types::AccessTokenId; use serde::Serialize; use serde_json::{Map, Value}; @@ -15,7 +19,7 @@ use crate::{ cli::{DiffArgs, DiffOutput, DiffResourceKind}, error::CliError, ops, - types::{DiffResource, S2BasinAndStreamUri, S2BasinUri, resolve_stream_config}, + types::{DiffResource, S2BasinAndStreamUri, S2BasinUri}, }; #[derive(Debug, Serialize)] @@ -294,7 +298,7 @@ fn resource_kind(resource: &DiffResource) -> DiffResourceKind { } } -/// Presentation of a stream configuration for comparison. +/// Presentation of an effective (fully resolved) stream configuration. /// /// - Field names and nesting mirror the API wire format so that `--output json` paths line up with /// API responses. @@ -302,7 +306,7 @@ fn resource_kind(resource: &DiffResource) -> DiffResourceKind { /// duration. #[derive(Serialize)] struct StreamConfigView { - storage_class: Option, + storage_class: String, retention_policy: String, timestamping: TimestampingConfigView, delete_on_empty: DeleteOnEmptyConfigView, @@ -319,9 +323,9 @@ struct DeleteOnEmptyConfigView { min_age: String, } -impl From for StreamConfigView { - fn from(config: StreamConfig) -> Self { - let StreamConfig { +impl From for StreamConfigView { + fn from(config: s2_common::config::StreamConfig) -> Self { + let s2_common::config::StreamConfig { storage_class, retention_policy, timestamping, @@ -330,7 +334,7 @@ impl From for StreamConfigView { let s2_common::config::TimestampingConfig { mode, uncapped } = timestamping; Self { - storage_class, + storage_class: wire_name(s2_api::v1::config::StorageClass::from(storage_class)), retention_policy: match retention_policy { s2_common::config::RetentionPolicy::Age(age) => compact_duration(age), s2_common::config::RetentionPolicy::Infinite() => "infinite".to_owned(), @@ -355,30 +359,25 @@ struct BasinConfigView { create_stream_on_read: bool, } -impl TryFrom for BasinConfigView { - type Error = s2_common::ValidationError; - - fn try_from(config: s2_api::v1::config::BasinConfig) -> Result { - let s2_api::v1::config::BasinConfig { +impl From for BasinConfigView { + fn from(config: s2_common::config::BasinConfig) -> Self { + let s2_common::config::BasinConfig { default_stream_config, stream_cipher, create_stream_on_append, create_stream_on_read, } = config; - Ok(Self { - default_stream_config: resolve_stream_config( - default_stream_config.unwrap_or_default(), - Default::default(), - )? - .into(), + Self { + default_stream_config: s2_common::config::StreamConfig::from(default_stream_config) + .into(), stream_cipher: match stream_cipher { None => "none".to_owned(), - Some(cipher) => wire_name(cipher), + Some(cipher) => wire_name(s2_api::v1::config::EncryptionAlgorithm::from(cipher)), }, create_stream_on_append, create_stream_on_read, - }) + } } } @@ -465,16 +464,20 @@ fn permissions_view(permissions: ReadWritePermissions) -> String { } fn basin_view(config: s2_api::v1::config::BasinConfig) -> Result { - Ok(serde_json::to_value(BasinConfigView::try_from(config)?)?) + let config: s2_common::config::BasinConfig = config.try_into()?; + Ok(serde_json::to_value(BasinConfigView::from(config))?) } /// Renders a basin's effective stream defaults for comparison against a concrete stream. fn basin_stream_defaults_view(config: s2_api::v1::config::BasinConfig) -> Result { - stream_view(config.default_stream_config.unwrap_or_default()) + let config: s2_common::config::BasinConfig = config.try_into()?; + let defaults = s2_common::config::StreamConfig::from(config.default_stream_config); + Ok(serde_json::to_value(StreamConfigView::from(defaults))?) } fn stream_view(config: s2_api::v1::config::StreamConfig) -> Result { - let config = resolve_stream_config(config, Default::default())?; + let config: s2_common::config::OptionalStreamConfig = config.try_into()?; + let config = s2_common::config::StreamConfig::from(config); Ok(serde_json::to_value(StreamConfigView::from(config))?) } @@ -738,7 +741,7 @@ mod tests { assert_eq!( view, json!({ - "storage_class": null, + "storage_class": "express", "retention_policy": "2d", "timestamping": {"mode": "client-prefer", "uncapped": false}, "delete_on_empty": {"min_age": "1h"} @@ -770,7 +773,7 @@ mod tests { fn omitted_and_explicit_defaults_render_identically() { let omitted = serde_json::from_value(json!({})).expect("empty config deserializes"); let explicit = serde_json::from_value(json!({ - "storage_class": null, + "storage_class": "express", "retention_policy": {"age": 604800}, "timestamping": {"mode": "client-prefer", "uncapped": false}, "delete_on_empty": {"min_age_secs": 0} @@ -799,7 +802,7 @@ mod tests { view, json!({ "default_stream_config": { - "storage_class": null, + "storage_class": "express", "retention_policy": "7d", "timestamping": {"mode": "client-prefer", "uncapped": false}, "delete_on_empty": {"min_age": "0s"} diff --git a/cli/src/main.rs b/cli/src/main.rs index 7c23f771..cc809202 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -42,7 +42,7 @@ use s2_sdk::{ S2, types::{ AppendRetryPolicy, CreateStreamInput, DeleteOnEmptyConfig, DeleteStreamInput, - EncryptionKey, LocationInfo, MeteredBytes, Metric, RetentionPolicy, RetryConfig, + EncryptionKey, MeteredBytes, Metric, RetentionPolicy, RetryConfig, StreamConfig as SdkStreamConfig, StreamName, TimestampingConfig, TimestampingMode, }, }; @@ -531,13 +531,13 @@ async fn run(cli: Cli) -> Result { Command::ListLocations => { let locations = ops::list_locations(&s2).await?; for location_info in locations { - print_location_listing(&location_info); + print_location_listing(location_info.name.to_string(), location_info.is_private); } } Command::GetDefaultLocation => { let location = ops::get_default_location(&s2).await?; - print_location_listing(&location); + print_location_listing(location.name.to_string(), location.is_private); } Command::SetDefaultLocation { location } => { @@ -549,7 +549,7 @@ async fn run(cli: Cli) -> Result { .green() .bold() ); - print_location_listing(&location); + print_location_listing(location.name.to_string(), location.is_private); } Command::GetAccountMetrics(args) => { @@ -657,13 +657,9 @@ async fn run(cli: Cli) -> Result { record_stream, args.uri, encryption_key.as_ref(), - ops::AppendOptions { - fencing_token: args.fencing_token, - match_seq_num: args.match_seq_num, - linger: *args.linger, - stream_config: (!args.stream_config.is_empty()) - .then(|| args.stream_config.into()), - }, + args.fencing_token, + args.match_seq_num, + *args.linger, ); let mut acks = std::pin::pin!(acks); let mut last_printed_batch_end: Option = None; @@ -852,7 +848,7 @@ async fn run(cli: Cli) -> Result { .with_mode(TimestampingMode::ClientRequire) .with_uncapped(true), ); - stream_config.storage_class = args.storage_class; + stream_config.storage_class = args.storage_class.map(Into::into); let s2 = S2::new(sdk_config.clone().with_retry( RetryConfig::new().with_append_retry_policy(AppendRetryPolicy::NoSideEffects), @@ -921,20 +917,9 @@ fn print_basin_listing(name: String, location: Option<&str>, is_deleting: bool) } } -fn print_location_listing(location: &LocationInfo) { - let visibility = format_location_visibility(location.is_private); - println!("{} {visibility}", location.name); - if let Some(storage_classes) = &location.storage_classes { - let classes = if storage_classes.is_empty() { - "none".to_owned() - } else { - storage_classes.join(", ") - }; - println!(" storage classes: {classes}"); - } - if let Some(default_storage_class) = &location.default_storage_class { - println!(" default storage class: {default_storage_class}"); - } +fn print_location_listing(name: String, is_private: bool) { + let visibility = format_location_visibility(is_private); + println!("{name} {visibility}"); } fn format_location_visibility(is_private: bool) -> colored::ColoredString { diff --git a/cli/src/ops.rs b/cli/src/ops.rs index 468d6ab0..7ae3218d 100644 --- a/cli/src/ops.rs +++ b/cli/src/ops.rs @@ -570,32 +570,23 @@ pub async fn read( stop = stop.with_until(..until); } - let mut input = ReadInput::new().with_start(start).with_stop(stop); - if !args.stream_config.is_empty() { - input = input.with_stream_config(args.stream_config.clone().into()); - } - stream - .read_session(input, ReadSessionConfig::default()) + .read_session( + ReadInput::new().with_start(start).with_stop(stop), + ReadSessionConfig::default(), + ) .await .map_err(|e| CliError::op(OpKind::Read, e)) } -/// Options controlling how records are appended. -pub struct AppendOptions { - pub fencing_token: Option, - pub match_seq_num: Option, - pub linger: Duration, - /// Stream configuration to apply if the stream is created on append. - pub stream_config: Option, -} - pub fn append<'a, S, E>( s2: &'a S2, records: S, uri: S2BasinAndStreamUri, encryption_key: Option<&'a EncryptionKey>, - options: AppendOptions, + fencing_token: Option, + match_seq_num: Option, + linger: Duration, ) -> impl Stream> + Send + 'a where S: Stream> + Send + Unpin + 'a, @@ -603,15 +594,12 @@ where { let stream = stream_with_encryption(s2, uri, encryption_key); - let batching_config = BatchingConfig::new().with_linger(options.linger); + let batching_config = BatchingConfig::new().with_linger(linger); let mut producer_config = ProducerConfig::new().with_batching(batching_config); - if let Some(config) = options.stream_config { - producer_config = producer_config.with_stream_config(config); - } - if let Some(ft) = options.fencing_token { + if let Some(ft) = fencing_token { producer_config = producer_config.with_fencing_token(ft); } - if let Some(seq) = options.match_seq_num { + if let Some(seq) = match_seq_num { producer_config = producer_config.with_match_seq_num(seq); } diff --git a/cli/src/types.rs b/cli/src/types.rs index d6187cbc..699a04b9 100644 --- a/cli/src/types.rs +++ b/cli/src/types.rs @@ -2,7 +2,6 @@ use std::{str::FromStr, time::Duration}; use clap::{Args, Parser, ValueEnum}; use colored::Colorize; -use compact_str::CompactString; use s2_sdk::{ self as sdk, types::{ @@ -175,7 +174,7 @@ pub struct BasinConfig { pub struct StreamConfig { #[arg(long)] /// Storage class for a stream. - pub storage_class: Option, + pub storage_class: Option, #[arg(long, help("Example: 1d, 1w, 1y"))] /// Retention policy for a stream. pub retention_policy: Option, @@ -202,16 +201,15 @@ impl StreamConfig { } } -pub fn resolve_stream_config( - config: s2_api::v1::config::StreamConfig, - basin_defaults: s2_api::v1::config::StreamConfig, -) -> Result { - let config: s2_common::config::OptionalStreamConfig = config.try_into()?; - Ok(config.merge(basin_defaults.try_into()?)) -} - pub use sdk::types::LocationName; +#[derive(ValueEnum, Debug, Clone, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum StorageClass { + Standard, + Express, +} + #[derive(ValueEnum, Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub enum TimestampingMode { @@ -304,7 +302,7 @@ impl From for sdk::types::StreamConfig { fn from(config: StreamConfig) -> Self { let mut stream_config = sdk::types::StreamConfig::new(); if let Some(storage_class) = config.storage_class { - stream_config = stream_config.with_storage_class(storage_class); + stream_config = stream_config.with_storage_class(storage_class.into()); } if let Some(retention_policy) = config.retention_policy { stream_config = stream_config.with_retention_policy(retention_policy.into()); @@ -319,6 +317,24 @@ impl From for sdk::types::StreamConfig { } } +impl From for sdk::types::StorageClass { + fn from(class: StorageClass) -> Self { + match class { + StorageClass::Standard => sdk::types::StorageClass::Standard, + StorageClass::Express => sdk::types::StorageClass::Express, + } + } +} + +impl From for StorageClass { + fn from(class: sdk::types::StorageClass) -> Self { + match class { + sdk::types::StorageClass::Standard => StorageClass::Standard, + sdk::types::StorageClass::Express => StorageClass::Express, + } + } +} + impl From for sdk::types::TimestampingMode { fn from(mode: TimestampingMode) -> Self { match mode { @@ -398,7 +414,7 @@ impl From for BasinConfig { impl From for StreamConfig { fn from(config: sdk::types::StreamConfig) -> Self { StreamConfig { - storage_class: config.storage_class, + storage_class: config.storage_class.map(Into::into), retention_policy: config.retention_policy.map(Into::into), timestamping: config.timestamping.map(Into::into), delete_on_empty: config.delete_on_empty.map(Into::into), @@ -410,7 +426,7 @@ impl From for sdk::types::StreamReconfiguration { fn from(config: StreamConfig) -> Self { let mut reconfig = sdk::types::StreamReconfiguration::new(); if let Some(storage_class) = config.storage_class { - reconfig = reconfig.with_storage_class(storage_class); + reconfig = reconfig.with_storage_class(storage_class.into()); } if let Some(retention_policy) = config.retention_policy { reconfig = reconfig.with_retention_policy(retention_policy.into()); @@ -481,35 +497,32 @@ where serializer.serialize_str(&humantime::format_duration(*value).to_string()) } -impl BasinMatcher { - /// Converts an SDK matcher, returning `None` for a matcher that matches no resources. - fn from_sdk(matcher: sdk::types::BasinMatcher) -> Option { +impl From for BasinMatcher { + fn from(matcher: sdk::types::BasinMatcher) -> Self { match matcher { - sdk::types::BasinMatcher::Exact(v) => Some(BasinMatcher::Exact(v)), - sdk::types::BasinMatcher::Prefix(v) => Some(BasinMatcher::Prefix(v)), - sdk::types::BasinMatcher::None => None, + sdk::types::BasinMatcher::Exact(v) => BasinMatcher::Exact(v), + sdk::types::BasinMatcher::Prefix(v) => BasinMatcher::Prefix(v), + sdk::types::BasinMatcher::None => BasinMatcher::Prefix(Default::default()), } } } -impl StreamMatcher { - /// Converts an SDK matcher, returning `None` for a matcher that matches no resources. - fn from_sdk(matcher: sdk::types::StreamMatcher) -> Option { +impl From for StreamMatcher { + fn from(matcher: sdk::types::StreamMatcher) -> Self { match matcher { - sdk::types::StreamMatcher::Exact(v) => Some(StreamMatcher::Exact(v)), - sdk::types::StreamMatcher::Prefix(v) => Some(StreamMatcher::Prefix(v)), - sdk::types::StreamMatcher::None => None, + sdk::types::StreamMatcher::Exact(v) => StreamMatcher::Exact(v), + sdk::types::StreamMatcher::Prefix(v) => StreamMatcher::Prefix(v), + sdk::types::StreamMatcher::None => StreamMatcher::Prefix(Default::default()), } } } -impl AccessTokenMatcher { - /// Converts an SDK matcher, returning `None` for a matcher that matches no resources. - fn from_sdk(matcher: sdk::types::AccessTokenMatcher) -> Option { +impl From for AccessTokenMatcher { + fn from(matcher: sdk::types::AccessTokenMatcher) -> Self { match matcher { - sdk::types::AccessTokenMatcher::Exact(v) => Some(AccessTokenMatcher::Exact(v)), - sdk::types::AccessTokenMatcher::Prefix(v) => Some(AccessTokenMatcher::Prefix(v)), - sdk::types::AccessTokenMatcher::None => None, + sdk::types::AccessTokenMatcher::Exact(v) => AccessTokenMatcher::Exact(v), + sdk::types::AccessTokenMatcher::Prefix(v) => AccessTokenMatcher::Prefix(v), + sdk::types::AccessTokenMatcher::None => AccessTokenMatcher::Prefix(Default::default()), } } } @@ -763,9 +776,9 @@ pub struct AccessTokenScope { impl From for AccessTokenScope { fn from(scope: sdk::types::AccessTokenScope) -> Self { AccessTokenScope { - basins: scope.basins.and_then(BasinMatcher::from_sdk), - streams: scope.streams.and_then(StreamMatcher::from_sdk), - access_tokens: scope.access_tokens.and_then(AccessTokenMatcher::from_sdk), + basins: scope.basins.map(Into::into), + streams: scope.streams.map(Into::into), + access_tokens: scope.access_tokens.map(Into::into), op_group_perms: scope.op_group_perms.map(Into::into), ops: scope.ops.into_iter().map(Operation::from).collect(), } @@ -933,37 +946,6 @@ mod tests { ); } - #[test] - fn match_none_scope_matchers_render_as_no_access() { - colored::control::set_override(false); - - // On the wire, an empty exact name means "match no resources". - let scope: s2_api::v1::access::AccessTokenScope = - serde_json::from_value(serde_json::json!({ - "basins": { "exact": "" }, - "streams": { "exact": "" }, - "access_tokens": { "exact": "" }, - })) - .unwrap(); - let scope = s2_sdk::types::AccessTokenScope::from(scope); - let info = AccessTokenInfo { - id: "tok".to_owned(), - expires_at: None, - auto_prefix_streams: false, - scope: scope.into(), - }; - - assert_eq!( - info.summary_block(3), - "tok expires never\n\ - \x20 basins=∅ streams=∅ tokens=∅ perms=none ops=0" - ); - let json = serde_json::to_value(&info.scope).unwrap(); - assert!(json["basins"].is_null(), "{json}"); - assert!(json["streams"].is_null(), "{json}"); - assert!(json["access_tokens"].is_null(), "{json}"); - } - #[test] fn op_group_perms_summary_omits_empty_groups() { assert_eq!( diff --git a/cli/tests/cli.rs b/cli/tests/cli.rs index 2baac327..9fdc0716 100644 --- a/cli/tests/cli.rs +++ b/cli/tests/cli.rs @@ -1,9 +1,7 @@ use std::{ - convert::Infallible, + io::{Read as _, Write as _}, net::TcpListener, - sync::{Arc, Mutex}, thread::JoinHandle, - time::Duration, }; use assert_cmd::Command; @@ -31,17 +29,31 @@ struct TestServer { } impl TestServer { - /// Serves one HTTP/2 request and returns the request line and headers it - /// saw. The client speaks h2 with prior knowledge over cleartext. fn start() -> Self { let listener = TcpListener::bind("127.0.0.1:0").expect("bind test server"); let endpoint = format!("http://{}", listener.local_addr().expect("server address")); let handle = std::thread::spawn(move || { - tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("test runtime") - .block_on(serve_one_request(listener)) + let (mut stream, _) = listener.accept().expect("accept request"); + let mut request = Vec::new(); + let mut buffer = [0_u8; 4096]; + loop { + let bytes_read = stream.read(&mut buffer).expect("read request"); + if bytes_read == 0 { + break; + } + request.extend_from_slice(&buffer[..bytes_read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + let body = r#"{"basins":[],"has_more":false}"#; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + .expect("write response"); + String::from_utf8(request).expect("request is UTF-8") }); Self { endpoint, handle } } @@ -51,55 +63,6 @@ impl TestServer { } } -const TEST_SERVER_TIMEOUT: Duration = Duration::from_secs(10); - -async fn serve_one_request(listener: TcpListener) -> String { - listener - .set_nonblocking(true) - .expect("non-blocking listener"); - let listener = tokio::net::TcpListener::from_std(listener).expect("tokio listener"); - let (stream, _) = tokio::time::timeout(TEST_SERVER_TIMEOUT, listener.accept()) - .await - .expect("timed out waiting for a connection") - .expect("accept connection"); - - let observed = Arc::new(Mutex::new(None)); - let captured = observed.clone(); - let service = hyper::service::service_fn(move |req: hyper::Request| { - let captured = captured.clone(); - async move { - let mut rendered = format!("{} {}\r\n", req.method(), req.uri()); - for (name, value) in req.headers() { - rendered.push_str(name.as_str()); - rendered.push_str(": "); - rendered.push_str(value.to_str().unwrap_or_default()); - rendered.push_str("\r\n"); - } - *captured.lock().expect("capture request") = Some(rendered); - - let body = r#"{"basins":[],"has_more":false}"#; - Ok::<_, Infallible>( - hyper::Response::builder() - .header("content-type", "application/json") - .body(http_body_util::Full::new(bytes::Bytes::from(body))) - .expect("build response"), - ) - } - }); - - // A client that exits right after its response can reset the connection, - // so the served result only matters when no request came through. - let served = tokio::time::timeout( - TEST_SERVER_TIMEOUT, - hyper::server::conn::http2::Builder::new(hyper_util::rt::TokioExecutor::new()) - .serve_connection(hyper_util::rt::TokioIo::new(stream), service), - ) - .await; - - let observed = observed.lock().expect("read request").take(); - observed.unwrap_or_else(|| panic!("server received no HTTP/2 request: {served:?}")) -} - impl TestEnv { fn new() -> Self { Self { diff --git a/cli/tests/integration.rs b/cli/tests/integration.rs index 7d0152f0..b193551f 100644 --- a/cli/tests/integration.rs +++ b/cli/tests/integration.rs @@ -563,95 +563,6 @@ fn append_from_stdin() { cleanup_stream(&basin, &stream); } -#[test] -#[serial] -fn append_with_stream_config() { - let basin = unique_name("test-cli-csoa-cfg"); - s2().args([ - "create-basin", - &basin, - "--retention-policy", - "7d", - "--create-stream-on-append", - ]) - .assert() - .success(); - wait_for_basin(&basin); - - let stream = unique_name("test-csoa-new"); - let uri = format!("s2://{basin}/{stream}"); - s2().args([ - "append", - &uri, - "--format", - "text", - "--input", - "-", - "--retention-policy", - "1h", - "--delete-on-empty-min-age", - "5m", - ]) - .write_stdin("first record\n") - .assert() - .success(); - - s2().args(["get-stream-config", &uri]) - .assert() - .success() - .stdout( - predicate::str::contains("1h") - .and(predicate::str::contains("5m")) - .and(predicate::str::contains("7days").not()), - ); - - s2().args([ - "append", - &uri, - "--format", - "text", - "--input", - "-", - "--retention-policy", - "2h", - ]) - .write_stdin("second record\n") - .assert() - .success(); - - s2().args(["get-stream-config", &uri]) - .assert() - .success() - .stdout(predicate::str::contains("1h").and(predicate::str::contains("2h").not())); - - cleanup_stream(&basin, &stream); - cleanup_basin(&basin); -} - -#[test] -#[serial] -fn read_with_stream_config() { - let basin = unique_name("test-cli-csor-cfg"); - s2().args(["create-basin", &basin, "--create-stream-on-read"]) - .assert() - .success(); - wait_for_basin(&basin); - - let stream = unique_name("test-csor-new"); - let uri = format!("s2://{basin}/{stream}"); - s2().args(["read", &uri, "--count", "1", "--retention-policy", "1h"]) - .assert() - .failure(); - - s2().args(["get-stream-config", &uri]) - .assert() - .success() - .stdout(predicate::str::contains("1h")); - - cleanup_stream(&basin, &stream); - cleanup_basin(&basin); -} - #[test] #[serial] fn tail_stream() { diff --git a/cli/tests/lite_wal.rs b/cli/tests/lite_wal.rs deleted file mode 100644 index 609ec24d..00000000 --- a/cli/tests/lite_wal.rs +++ /dev/null @@ -1,227 +0,0 @@ -use std::{ - fs::{self, File}, - io::{Read, Write}, - net::{SocketAddr, TcpListener, TcpStream}, - path::Path, - process::{Child, Command as ProcessCommand, Stdio}, - time::{Duration, Instant}, -}; - -use assert_cmd::Command; -use tempfile::TempDir; - -fn base_command(home: &Path) -> ProcessCommand { - let mut cmd = ProcessCommand::new(assert_cmd::cargo::cargo_bin!("s2")); - cmd.env_clear() - .env("HOME", home) - .env("XDG_CONFIG_HOME", home.join(".config")) - .env("APPDATA", home) - .env("USERPROFILE", home) - .env("NO_COLOR", "1") - .env("RUST_LOG", "warn"); - cmd -} - -fn command(home: &Path) -> Command { - let mut cmd = Command::from_std(base_command(home)); - cmd.timeout(Duration::from_secs(30)); - cmd -} - -#[test] -fn wal_store_requires_a_persistent_main_store_and_one_backend() { - let home = tempfile::tempdir().unwrap(); - for args in [ - vec!["lite", "--wal-bucket", "wal"], - vec!["lite", "--wal-local-root", "wal"], - vec![ - "lite", - "--bucket", - "main", - "--wal-bucket", - "wal", - "--wal-local-root", - "wal", - ], - vec!["lite", "--bucket", "main", "--local-root", "main"], - ] { - command(home.path()).args(args).assert().failure().code(2); - } - command(home.path()) - .env("S2LITE_WAL_BUCKET", "wal") - .args(["lite"]) - .assert() - .failure() - .code(2); -} - -struct Server { - child: Child, - address: SocketAddr, -} - -impl Server { - fn start(root: &Path, attempt: &str) -> Self { - for bind_attempt in 0.. { - let log_path = root.join(format!("server-{attempt}-{bind_attempt}.log")); - match Self::spawn(root, &log_path) { - Ok(server) => return server, - Err(log) if bind_attempt < 5 && log.contains("Address already in use") => { - continue; - } - Err(log) => panic!("Server exited before becoming healthy: {log}"), - } - } - unreachable!() - } - - fn spawn(root: &Path, log_path: &Path) -> Result { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let address = listener.local_addr().unwrap(); - drop(listener); - let log = File::create(log_path).unwrap(); - let mut cmd = base_command(root); - cmd.args([ - "lite", - "--path", - "db", - "--port", - &address.port().to_string(), - "--local-root", - ]) - .arg(root.join("main")) - .env("S2LITE_WAL_LOCAL_ROOT", root.join("wal")) - .env("SL8_FLUSH_INTERVAL", "1ms") - .env("SL8_L0_SST_SIZE_BYTES", "65536"); - let child = cmd - .stdin(Stdio::null()) - .stdout(log.try_clone().unwrap()) - .stderr(log) - .spawn() - .unwrap(); - let mut server = Self { child, address }; - let deadline = Instant::now() + Duration::from_secs(30); - loop { - if server.child.try_wait().unwrap().is_some() { - return Err(fs::read_to_string(log_path).unwrap()); - } - if let Ok(mut stream) = TcpStream::connect_timeout(&address, Duration::from_millis(100)) - { - stream - .set_read_timeout(Some(Duration::from_secs(1))) - .unwrap(); - stream - .set_write_timeout(Some(Duration::from_secs(1))) - .unwrap(); - let mut response = [0; 256]; - if stream - .write_all( - b"GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", - ) - .is_ok() - && let Ok(n) = stream.read(&mut response) - && response[..n].starts_with(b"HTTP/1.1 200") - { - return Ok(server); - } - } - assert!( - Instant::now() < deadline, - "Server did not become healthy: {}", - fs::read_to_string(log_path).unwrap() - ); - std::thread::sleep(Duration::from_millis(25)); - } - } - - fn client(&self, root: &Path, args: &[&str], input: Option<&str>) -> String { - let endpoint = format!("http://{}", self.address); - let mut cmd = command(root); - cmd.env("S2_ACCOUNT_ENDPOINT", &endpoint) - .env("S2_BASIN_ENDPOINT", &endpoint) - .env("S2_ACCESS_TOKEN", "test") - .args(args); - if let Some(input) = input { - cmd.write_stdin(input); - } - let output = cmd.output().unwrap(); - assert!( - output.status.success(), - "{args:?}: {}", - String::from_utf8_lossy(&output.stderr) - ); - String::from_utf8(output.stdout).unwrap() - } - - fn crash(&mut self) { - self.child.kill().unwrap(); - self.child.wait().unwrap(); - } -} - -impl Drop for Server { - fn drop(&mut self) { - let _ = self.child.kill(); - let _ = self.child.wait(); - } -} - -fn has_sst(path: &Path) -> bool { - fs::read_dir(path).is_ok_and(|entries| { - entries - .flatten() - .any(|entry| entry.path().extension().is_some_and(|ext| ext == "sst")) - }) -} - -#[test] -fn separate_wal_routes_files_and_recovers_acknowledged_records_after_restart() { - let root: TempDir = tempfile::tempdir().unwrap(); - let mut server = Server::start(root.path(), "before"); - server.client(root.path(), &["create-basin", "wal-test"], None); - server.client( - root.path(), - &["create-stream", "s2://wal-test/recovery"], - None, - ); - let mut input = (0..64) - .map(|i| format!("record-{i}-{}\n", "x".repeat(4096))) - .collect::(); - server.client( - root.path(), - &["append", "s2://wal-test/recovery"], - Some(&input), - ); - let deadline = Instant::now() + Duration::from_secs(10); - while !has_sst(&root.path().join("main/db/compacted")) { - assert!(Instant::now() < deadline, "Main-store SST was not flushed"); - std::thread::sleep(Duration::from_millis(25)); - } - // Leave a final small write in the WAL after the earlier data reaches L0. - let last = "last-acknowledged-record\n"; - server.client( - root.path(), - &["append", "s2://wal-test/recovery"], - Some(last), - ); - input.push_str(last); - assert!(has_sst(&root.path().join("wal/db/wal"))); - assert!(!root.path().join("main/db/wal").exists()); - assert!(!root.path().join("wal/db/compacted").exists()); - assert!(!root.path().join("wal/db/manifest").exists()); - server.crash(); - let restarted = Server::start(root.path(), "after"); - let read = restarted.client( - root.path(), - &[ - "read", - "s2://wal-test/recovery", - "--seq-num", - "0", - "--count", - "65", - ], - None, - ); - assert_eq!(read, input); -} diff --git a/common/CHANGELOG.md b/common/CHANGELOG.md index 8fa9a24a..5d7db4c4 100644 --- a/common/CHANGELOG.md +++ b/common/CHANGELOG.md @@ -2,30 +2,6 @@ All notable changes to this project will be documented in this file. -## [0.42.0] - 2026-09-25 - -### Features - -- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) - - - -## [0.41.3] - 2026-09-22 - -### Miscellaneous Tasks - -- Update Cargo.toml dependencies - - - -## [0.41.2] - 2026-09-10 - -### Bug Fixes - -- Reject NUL bytes in stream names and access token IDs ([#728](https://github.com/s2-streamstore/s2/issues/728)) - - - ## [0.41.1] - 2026-07-22 ### Miscellaneous Tasks diff --git a/common/Cargo.toml b/common/Cargo.toml index acdd4354..202963a7 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-common" -version = "0.42.0" +version = "0.41.1" description = "Common stuff for client and servers for S2, the durable streams API" edition.workspace = true license.workspace = true diff --git a/common/src/access.rs b/common/src/access.rs index 5d67b948..01b3749f 100644 --- a/common/src/access.rs +++ b/common/src/access.rs @@ -30,12 +30,6 @@ impl AccessTokenIdStr { ); } - if id.contains('\0') { - return Err( - format!("access token {} must not contain NUL bytes", T::FIELD_NAME).into(), - ); - } - if id.len() > caps::MAX_ACCESS_TOKEN_ID_LEN { return Err(format!( "access token {} must not exceed {} bytes in length", @@ -270,7 +264,6 @@ mod test { #[case::dot(".".to_owned())] #[case::dot_dot("..".to_owned())] #[case::too_long("a".repeat(crate::caps::MAX_ACCESS_TOKEN_ID_LEN + 1))] - #[case::nul("a\0b".to_owned())] fn validate_id_err(#[case] id: String) { AccessTokenIdStr::::validate_str(&id).expect_err("expected validation error"); } @@ -289,7 +282,6 @@ mod test { #[rstest] #[case::too_long("a".repeat(crate::caps::MAX_ACCESS_TOKEN_ID_LEN + 1))] - #[case::nul("a\0b".to_owned())] fn validate_prefix_err(#[case] prefix: String) { AccessTokenIdStr::::validate_str(&prefix) .expect_err("expected validation error"); @@ -309,7 +301,6 @@ mod test { #[rstest] #[case::too_long("a".repeat(crate::caps::MAX_ACCESS_TOKEN_ID_LEN + 1))] - #[case::nul("a\0b".to_owned())] fn validate_start_after_err(#[case] start_after: String) { AccessTokenIdStr::::validate_str(&start_after) .expect_err("expected validation error"); diff --git a/common/src/basin.rs b/common/src/basin.rs index 9538ce4f..255ce2f4 100644 --- a/common/src/basin.rs +++ b/common/src/basin.rs @@ -231,7 +231,6 @@ mod test { #[case::invalid_first_char("Abcdefgh".to_owned())] #[case::invalid_last_char("abcdefg-".to_owned())] #[case::invalid_characters("abcd_efg".to_owned())] - #[case::nul("abcd\0efg".to_owned())] fn validate_name_err(#[case] name: String) { BasinNameStr::::validate_str(&name).expect_err("expected validation error"); } @@ -249,7 +248,6 @@ mod test { #[case::too_long("a".repeat(crate::caps::MAX_BASIN_NAME_LEN + 1))] #[case::invalid_first_char("-abc".to_owned())] #[case::invalid_characters("ab_cd".to_owned())] - #[case::nul("ab\0cd".to_owned())] fn validate_prefix_err(#[case] prefix: String) { BasinNameStr::::validate_str(&prefix).expect_err("expected validation error"); } @@ -269,7 +267,6 @@ mod test { #[case::too_long("a".repeat(crate::caps::MAX_BASIN_NAME_LEN + 1))] #[case::invalid_first_char("-abc".to_owned())] #[case::invalid_characters("ab_cd".to_owned())] - #[case::nul("ab\0cd".to_owned())] fn validate_start_after_err(#[case] start_after: String) { BasinNameStr::::validate_str(&start_after) .expect_err("expected validation error"); diff --git a/common/src/config.rs b/common/src/config.rs index 8fc17e45..83526ec2 100644 --- a/common/src/config.rs +++ b/common/src/config.rs @@ -2,9 +2,8 @@ //! //! Stream configuration uses three representations: //! -//! - Merged (`StreamConfig`, `TimestampingConfig`, `DeleteOnEmptyConfig`): values produced by -//! merging optional configs with defaults using `merge()`. Storage class remains unspecified when -//! neither the stream nor basin supplies one. +//! - Resolved (`StreamConfig`, `TimestampingConfig`, `DeleteOnEmptyConfig`): concrete values, +//! produced by merging optional configs with defaults using `merge()`. //! //! - Optional (`OptionalStreamConfig`, `OptionalTimestampingConfig`, //! `OptionalDeleteOnEmptyConfig`): partial configuration layers, where `None` means "not set at @@ -18,18 +17,40 @@ //! applies the inner reconfiguration to the existing value, while `Specified(None)` //! clears it to the default. //! -//! `merge()` applies configuration layers with precedence: -//! stream-level → basin-level → field default. +//! `merge()` resolves optional configs into resolved configs with precedence: +//! stream-level → basin-level → system default (via `Option::or` chaining). //! //! Basin config also carries basin-level knobs like `stream_cipher`, //! `create_stream_on_append`, and `create_stream_on_read`. use std::time::Duration; -use compact_str::CompactString; - use crate::{ValidationError, encryption::EncryptionAlgorithm, maybe::Maybe}; +#[derive( + Debug, + Default, + Clone, + Copy, + strum::Display, + strum::IntoStaticStr, + strum::EnumIter, + strum::FromRepr, + strum::EnumString, + PartialEq, + Eq, + Hash, +)] +#[cfg_attr(feature = "clap", derive(clap::ValueEnum))] +#[repr(u8)] +pub enum StorageClass { + #[strum(serialize = "standard")] + Standard = 1, + #[default] + #[strum(serialize = "express")] + Express = 2, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RetentionPolicy { Age(Duration), @@ -89,7 +110,7 @@ impl DeleteOnEmptyConfig { #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct StreamConfig { - pub storage_class: Option, + pub storage_class: StorageClass, pub retention_policy: RetentionPolicy, pub timestamping: TimestampingConfig, pub delete_on_empty: DeleteOnEmptyConfig, @@ -108,7 +129,7 @@ pub struct DeleteOnEmptyReconfiguration { #[derive(Debug, Clone, Default)] pub struct StreamReconfiguration { - pub storage_class: Maybe>, + pub storage_class: Maybe>, pub retention_policy: Maybe>, pub timestamping: Maybe>, pub delete_on_empty: Maybe>, @@ -196,7 +217,7 @@ impl From for OptionalDeleteOnEmptyConfig { #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct OptionalStreamConfig { - pub storage_class: Option, + pub storage_class: Option, pub retention_policy: Option, pub timestamping: OptionalTimestampingConfig, pub delete_on_empty: OptionalDeleteOnEmptyConfig, @@ -237,7 +258,10 @@ impl OptionalStreamConfig { } pub fn merge(self, basin_defaults: Self) -> StreamConfig { - let storage_class = self.storage_class.or(basin_defaults.storage_class); + let storage_class = self + .storage_class + .or(basin_defaults.storage_class) + .unwrap_or_default(); let retention_policy = self .retention_policy @@ -267,7 +291,7 @@ impl From for StreamConfig { } = value; Self { - storage_class, + storage_class: storage_class.unwrap_or_default(), retention_policy: retention_policy.unwrap_or_default(), timestamping: timestamping.into(), delete_on_empty: delete_on_empty.into(), @@ -285,7 +309,7 @@ impl From for OptionalStreamConfig { } = value; Self { - storage_class, + storage_class: Some(storage_class), retention_policy: Some(retention_policy), timestamping: timestamping.into(), delete_on_empty: delete_on_empty.into(), diff --git a/common/src/location.rs b/common/src/location.rs index 86605447..ca07367f 100644 --- a/common/src/location.rs +++ b/common/src/location.rs @@ -146,8 +146,6 @@ impl From for CompactString { pub struct LocationInfo { pub name: LocationName, pub is_private: bool, - pub storage_classes: Vec, - pub default_storage_class: CompactString, } #[cfg(test)] @@ -175,7 +173,6 @@ mod test { #[case::slash("aws/us-east-1".to_owned())] #[case::space("aws:us east-1".to_owned())] #[case::multibyte("aws:é".to_owned())] - #[case::nul("aws:us\0east-1".to_owned())] fn validate_name_err(#[case] location: String) { location .parse::() diff --git a/common/src/stream.rs b/common/src/stream.rs index 8f6df6f8..6ed7be49 100644 --- a/common/src/stream.rs +++ b/common/src/stream.rs @@ -34,10 +34,6 @@ impl StreamNameStr { return Err(format!("stream {} must not be \".\" or \"..\"", T::FIELD_NAME).into()); } - if name.contains('\0') { - return Err(format!("stream {} must not contain NUL bytes", T::FIELD_NAME).into()); - } - if name.len() > caps::MAX_STREAM_NAME_LEN { return Err(format!( "stream {} must not exceed {} bytes in length", @@ -390,8 +386,6 @@ mod test { #[rstest] #[case::normal("my-stream".to_owned())] - #[case::control_chars("a\tb\nc\rd\x01e".to_owned())] - #[case::unicode("stream/名前 😀?#%20".to_owned())] #[case::max_len("a".repeat(crate::caps::MAX_STREAM_NAME_LEN))] fn validate_name_ok(#[case] name: String) { assert_eq!(StreamNameStr::::validate_str(&name), Ok(())); @@ -402,10 +396,6 @@ mod test { #[case::dot(".".to_owned())] #[case::dot_dot("..".to_owned())] #[case::too_long("a".repeat(crate::caps::MAX_STREAM_NAME_LEN + 1))] - #[case::nul("a\0b".to_owned())] - #[case::leading_nul("\0a".to_owned())] - #[case::trailing_nul("a\0".to_owned())] - #[case::only_nul("\0".to_owned())] fn validate_name_err(#[case] name: String) { StreamNameStr::::validate_str(&name).expect_err("expected validation error"); } @@ -421,8 +411,6 @@ mod test { #[rstest] #[case::too_long("a".repeat(crate::caps::MAX_STREAM_NAME_LEN + 1))] - #[case::nul("a\0b".to_owned())] - #[case::only_nul("\0".to_owned())] fn validate_prefix_err(#[case] prefix: String) { StreamNameStr::::validate_str(&prefix).expect_err("expected validation error"); } @@ -441,8 +429,6 @@ mod test { #[rstest] #[case::too_long("a".repeat(crate::caps::MAX_STREAM_NAME_LEN + 1))] - #[case::nul("a\0b".to_owned())] - #[case::only_nul("\0".to_owned())] fn validate_start_after_err(#[case] start_after: String) { StreamNameStr::::validate_str(&start_after) .expect_err("expected validation error"); diff --git a/deny.toml b/deny.toml index 24aac6b4..48b28369 100644 --- a/deny.toml +++ b/deny.toml @@ -6,6 +6,8 @@ ignore = [ # Transitive deps via slatedb/foyer — not actionable. "RUSTSEC-2024-0436", # paste unmaintained "RUSTSEC-2025-0141", # bincode unmaintained + # Transitive dep via tabled_derive — no fixed release available. + "RUSTSEC-2026-0173", # proc-macro-error2 unmaintained # Transitive dep via object_store (slatedb) — fix requires object_store >=0.13. "RUSTSEC-2026-0194", # quick-xml quadratic attribute check "RUSTSEC-2026-0195", # quick-xml unbounded namespace allocation diff --git a/justfile b/justfile index 7fbc7058..fb607cff 100644 --- a/justfile +++ b/justfile @@ -12,15 +12,15 @@ build *args: sync # Run clippy linter clippy *args: sync - cargo clippy --locked --workspace --all-features --all-targets {{args}} -- -D warnings --allow deprecated + cargo clippy --workspace --all-features --all-targets {{args}} -- -D warnings --allow deprecated # Run clippy on the simulator (separate workspace) sim-clippy *args: - RUSTFLAGS="--cfg tokio_unstable" cargo clippy --manifest-path sim/Cargo.toml --all-targets {{args}} -- -D warnings --allow deprecated + RUSTFLAGS="--cfg tokio_unstable" cargo clippy --locked --manifest-path sim/Cargo.toml --all-targets {{args}} -- -D warnings --allow deprecated # Ensure cargo-deny is installed _ensure-deny: - @cargo deny --version > /dev/null 2>&1 || (echo "cargo-deny is required; run: brew install cargo-deny" && exit 1) + @cargo deny --version > /dev/null 2>&1 || cargo install cargo-deny # Run cargo-deny checks deny *args: _ensure-deny @@ -37,24 +37,25 @@ fmt: _ensure-nightly # Ensure cargo-nextest is installed _ensure-nextest: - @cargo nextest --version > /dev/null 2>&1 || (echo "cargo-nextest is required; run: brew install cargo-nextest" && exit 1) + @cargo nextest --version > /dev/null 2>&1 || cargo install cargo-nextest # Run tests with nextest (excludes Docker-backed and live integration tests) test *args: sync _ensure-nextest - cargo nextest run --locked --workspace --all-features --exclude s2-testcontainers -E 'not ((package(s2-cli) & binary(integration)) or (package(s2-sdk) & (binary(account_ops) or binary(basin_ops) or binary(metrics_ops) or binary(stream_ops))))' {{args}} + cargo nextest run --workspace --all-features --exclude s2-testcontainers -E 'not ((package(s2-cli) & binary(integration)) or (package(s2-sdk) & (binary(account_ops) or binary(basin_ops) or binary(metrics_ops) or binary(stream_ops))))' {{args}} # Run CLI integration tests (requires s2 lite server running) test-cli-integration: sync _ensure-nextest S2_ACCESS_TOKEN=test S2_ACCOUNT_ENDPOINT=http://localhost S2_BASIN_ENDPOINT=http://localhost \ - cargo nextest run --locked -p s2-cli --test integration + cargo nextest run -p s2-cli --test integration # Run SDK integration tests (requires S2_ACCESS_TOKEN and optional custom endpoints) test-sdk-integration: sync _ensure-nextest - cargo nextest run --locked -p s2-sdk --test account_ops --test basin_ops --test metrics_ops --test stream_ops + cargo nextest run -p s2-sdk --test account_ops --test basin_ops --test metrics_ops --test stream_ops # Verify Cargo.lock is up-to-date check-locked: cargo metadata --locked --format-version 1 >/dev/null + cargo metadata --locked --manifest-path sim/Cargo.toml --format-version 1 >/dev/null # Install git hooks from hooks/ install-hooks: @@ -67,7 +68,7 @@ clean: # Run s2-lite lite *args: - cargo run --locked --release -p s2-cli -- lite {{args}} + cargo run --release -p s2-cli -- lite {{args}} # Run the s2-lite deterministic simulation (e.g. `just sim smoke --seed 42`, # `just sim linearizable --seed 42 --clients 3 --ops-per-client 100`) @@ -75,4 +76,4 @@ lite *args: # dependencies (getrandom fork, etc.) from the main workspace. # tokio_unstable is required so turmoil can seed tokio's internal RNG. sim *args: - RUSTFLAGS="--cfg tokio_unstable" cargo run --manifest-path sim/Cargo.toml --profile sim -- {{args}} + RUSTFLAGS="--cfg tokio_unstable" cargo run --locked --manifest-path sim/Cargo.toml --profile sim -- {{args}} diff --git a/lite/CHANGELOG.md b/lite/CHANGELOG.md index 4f80ead3..6480c3e8 100644 --- a/lite/CHANGELOG.md +++ b/lite/CHANGELOG.md @@ -2,89 +2,6 @@ All notable changes to this project will be documented in this file. -## [0.43.0] - 2026-09-25 - -### Features - -- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) - -### Bug Fixes - -- Gate basin deletion draining on per-basin progress ([#779](https://github.com/s2-streamstore/s2/issues/779)) - - - -## [0.42.14] - 2026-09-24 - -### Bug Fixes - -- Coalesce delete-on-empty scheduling ([#772](https://github.com/s2-streamstore/s2/issues/772)) - - - -## [0.42.13] - 2026-09-22 - -### Features - -- Support a separate WAL object store ([#766](https://github.com/s2-streamstore/s2/issues/766)) - -### Bug Fixes - -- Keep streamers alive until pending writes settle ([#757](https://github.com/s2-streamstore/s2/issues/757)) -- Await durable deletion markers on retries ([#756](https://github.com/s2-streamstore/s2/issues/756)) -- Prevent stale or missed stream config updates ([#759](https://github.com/s2-streamstore/s2/issues/759)) -- Apply the at-tail read guard after resolving a timestamp start ([#762](https://github.com/s2-streamstore/s2/issues/762)) -- Prevent deletion races when recreating streams ([#760](https://github.com/s2-streamstore/s2/issues/760)) -- Preserve SSE read budgets across reconnects ([#727](https://github.com/s2-streamstore/s2/issues/727)) - -### Performance - -- Avoid copying serialized append buffers ([#763](https://github.com/s2-streamstore/s2/issues/763)) -- Reuse acknowledgement queue capacity ([#764](https://github.com/s2-streamstore/s2/issues/764)) - - - -## [0.42.12] - 2026-09-16 - -### Miscellaneous Tasks - -- Update Cargo.lock dependencies - - - -## [0.42.11] - 2026-09-11 - -### Features - -- `s2-stream-config` header for auto-created streams ([#718](https://github.com/s2-streamstore/s2/issues/718)) - - - -## [0.42.10] - 2026-09-11 - -### Miscellaneous Tasks - -- Update Cargo.lock dependencies - - - -## [0.42.9] - 2026-09-10 - -### Bug Fixes - -- Close SlateDB on graceful shutdown -- Reject NUL bytes in stream names and access token IDs ([#728](https://github.com/s2-streamstore/s2/issues/728)) - - - -## [0.42.8] - 2026-09-01 - -### Miscellaneous Tasks - -- Update Cargo.lock dependencies - - - ## [0.42.7] - 2026-08-18 ### Miscellaneous Tasks diff --git a/lite/Cargo.toml b/lite/Cargo.toml index 8f0d8988..5003a940 100644 --- a/lite/Cargo.toml +++ b/lite/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-lite" -version = "0.43.0" +version = "0.42.7" description = "Lightweight server implementation of S2, the durable streams API, backed by object storage" edition.workspace = true license.workspace = true diff --git a/lite/src/backend/append.rs b/lite/src/backend/append.rs index aa83f752..82343edb 100644 --- a/lite/src/backend/append.rs +++ b/lite/src/backend/append.rs @@ -7,7 +7,6 @@ use std::{ use futures::{Stream, StreamExt as _, future::OptionFuture, stream::FuturesOrdered}; use s2_common::{ basin::BasinName, - config::OptionalStreamConfig, encryption::{EncryptionKey, EncryptionSpec}, record::{SeqNum, StreamPosition}, stream::{AppendAck, AppendInput, StreamName}, @@ -15,26 +14,20 @@ use s2_common::{ use s2_storage::record::encrypt_append_input; use tokio::sync::oneshot; -use super::{Backend, StreamHandle, core::AutoCreateOn}; +use super::{Backend, StreamHandle}; use crate::backend::error::{AppendError, AppendErrorInternal, StorageError}; impl Backend { - /// Open a stream for an append or append session. - /// - /// `stream_config` is applied if the stream is created on append. Unset fields inherit the - /// basin's default stream configuration. Ignored if the stream already exists. pub async fn open_for_append( &self, basin: &BasinName, stream: &StreamName, encryption_key: Option, - stream_config: OptionalStreamConfig, ) -> Result { self.stream_handle_with_auto_create::( basin, stream, - AutoCreateOn::Append, - stream_config, + |config| config.create_stream_on_append, |cipher| Ok(EncryptionSpec::resolve(cipher, encryption_key)?), ) .await @@ -141,8 +134,6 @@ pub fn admit( } } -const MIN_PENDING_APPENDS_CAPACITY: usize = 16; - #[derive(Debug, Default)] pub struct PendingAppends { queue: VecDeque, @@ -197,12 +188,8 @@ impl PendingAppends { } // Lots of small appends could cause this, // as we bound only on total bytes not num batches. - // Keep a small buffer to reuse across ordinary drain/refill cycles. - if self.queue.capacity() > MIN_PENDING_APPENDS_CAPACITY - && self.queue.capacity() >= 4 * self.queue.len() - { - self.queue - .shrink_to((self.queue.len() * 2).max(MIN_PENDING_APPENDS_CAPACITY)); + if self.queue.capacity() >= 4 * self.queue.len() { + self.queue.shrink_to(self.queue.len() * 2); } } @@ -284,115 +271,3 @@ impl BlockedReplySender { let _ = self.tx.send(reply); } } - -#[cfg(test)] -mod tests { - use super::*; - - fn position(seq_num: SeqNum) -> StreamPosition { - StreamPosition { - seq_num, - timestamp: seq_num, - } - } - - fn accept( - pending: &mut PendingAppends, - seq_num: SeqNum, - ) -> oneshot::Receiver> { - let (tx, rx) = oneshot::channel(); - let ticket = admit(tx, None).expect("open receiver"); - pending.accept(ticket, position(seq_num)..position(seq_num + 1)); - rx - } - - #[rstest::rstest] - #[case(1)] - #[case(16)] - fn small_append_bursts_reuse_queue_capacity(#[case] burst_size: SeqNum) { - let mut pending = PendingAppends::new(); - let mut retained_capacity = None; - for burst in 0..4 { - let start = burst * burst_size; - let receivers: Vec<_> = (start..start + burst_size) - .map(|seq_num| accept(&mut pending, seq_num)) - .collect(); - let capacity = *retained_capacity.get_or_insert(pending.queue.capacity()); - assert_eq!(pending.queue.capacity(), capacity); - - // The streamer advances durability one append at a time, even when - // the whole burst became durable in the same WAL flush. - for (offset, mut rx) in receivers.into_iter().enumerate() { - let seq_num = start + offset as SeqNum; - assert!(matches!( - rx.try_recv(), - Err(oneshot::error::TryRecvError::Empty) - )); - pending.on_stable(position(seq_num + 1)); - let ack = rx - .try_recv() - .expect("ack delivered") - .expect("append accepted"); - assert_eq!(ack.start, position(seq_num)); - assert_eq!(ack.end, position(seq_num + 1)); - assert_eq!(ack.tail, position(seq_num + 1)); - assert_eq!(pending.queue.capacity(), capacity); - } - assert!(pending.queue.is_empty()); - } - } - - #[test] - fn unused_queue_stays_unallocated() { - let mut pending = PendingAppends::new(); - pending.on_stable(StreamPosition::MIN); - assert_eq!(pending.queue.capacity(), 0); - } - - #[test] - fn large_append_burst_releases_excess_capacity() { - let mut pending = PendingAppends::new(); - let mut receivers: Vec<_> = (0..128) - .map(|seq_num| accept(&mut pending, seq_num)) - .collect(); - let peak_capacity = pending.queue.capacity(); - - pending.on_stable(position(96)); - assert_eq!(pending.queue.len(), 32); - let reduced_capacity = pending.queue.capacity(); - assert!(reduced_capacity < peak_capacity); - for (seq_num, rx) in receivers.iter_mut().enumerate() { - if seq_num < 96 { - let ack = rx - .try_recv() - .expect("ack delivered") - .expect("append accepted"); - assert_eq!(ack.start, position(seq_num as SeqNum)); - assert_eq!(ack.end, position(seq_num as SeqNum + 1)); - assert_eq!(ack.tail, position(96)); - } else { - assert!(matches!( - rx.try_recv(), - Err(oneshot::error::TryRecvError::Empty) - )); - } - } - - // A small change in occupancy should not immediately shrink again. - pending.on_stable(position(97)); - assert_eq!(pending.queue.capacity(), reduced_capacity); - pending.on_stable(position(128)); - assert!(pending.queue.is_empty()); - assert!(pending.queue.capacity() > 0); - assert!(pending.queue.capacity() <= MIN_PENDING_APPENDS_CAPACITY); - for (seq_num, rx) in receivers.iter_mut().enumerate().skip(96) { - let ack = rx - .try_recv() - .expect("ack delivered") - .expect("append accepted"); - assert_eq!(ack.start, position(seq_num as SeqNum)); - assert_eq!(ack.end, position(seq_num as SeqNum + 1)); - assert_eq!(ack.tail, position(if seq_num == 96 { 97 } else { 128 })); - } - } -} diff --git a/lite/src/backend/basins.rs b/lite/src/backend/basins.rs index 73f1cb01..c37511af 100644 --- a/lite/src/backend/basins.rs +++ b/lite/src/backend/basins.rs @@ -11,11 +11,7 @@ use slatedb::{ }; use time::OffsetDateTime; -use super::{ - Backend, - bgtasks::BgtaskTrigger, - store::{db_txn_commit_durable, db_txn_get, db_txn_get_with}, -}; +use super::{Backend, bgtasks::BgtaskTrigger, store::db_txn_get}; use crate::backend::{ error::{ BasinAlreadyExistsError, BasinDeletionPendingError, BasinNotFoundError, DeleteBasinError, @@ -67,8 +63,6 @@ impl Backend { Ok(Page::new(basins, has_more)) } - /// Any outcome asserting the basin exists — `Created`, `Updated`, `Noop`, - /// or `BasinAlreadyExists` — is readable at `DurabilityLevel::Remote`. pub async fn provision_basin( &self, basin: BasinName, @@ -79,12 +73,7 @@ impl Backend { let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; - // A transaction can see metadata that has not been flushed yet. - let (existing_meta, existing_seq) = db_txn_get_with(&txn, &meta_key, |entry| { - Ok((kv::basin_meta::deser_value(entry.value)?, entry.seq)) - }) - .await? - .unzip(); + let existing_meta = db_txn_get(&txn, &meta_key, kv::basin_meta::deser_value).await?; if let Some(existing_meta) = &existing_meta && existing_meta.deleted_at.is_some() { @@ -96,7 +85,7 @@ impl Backend { let new_creation_idempotency_key = request_token .as_ref() .map(|req_token| creation_idempotency_key(req_token, &config)); - let result = if new_creation_idempotency_key.is_some() + return if new_creation_idempotency_key.is_some() && existing.creation_idempotency_key == new_creation_idempotency_key { Ok(ProvisionResult::Noop(BasinInfo { @@ -108,10 +97,6 @@ impl Backend { } else { Err(BasinAlreadyExistsError { basin }.into()) }; - drop(txn); - self.await_durable_seq(existing_seq.expect("existing meta was read")) - .await?; - return result; } (Some(existing), ProvisionMode::Ensure) => { let meta = kv::basin_meta::BasinMeta { @@ -145,15 +130,11 @@ impl Backend { }), }; - if matches!(&outcome, ProvisionResult::Noop(_)) { - drop(txn); - self.await_durable_seq(existing_seq.expect("noop implies existing meta")) - .await?; - } else { + if !matches!(&outcome, ProvisionResult::Noop(_)) { let meta = outcome.inner(); txn.put(&meta_key, kv::basin_meta::ser_value(meta))?; - db_txn_commit_durable(txn).await?; + txn.commit().await?; } Ok(outcome.map(|meta| BasinInfo { @@ -198,7 +179,7 @@ impl Backend { txn.put(&meta_key, kv::basin_meta::ser_value(&meta))?; - db_txn_commit_durable(txn).await?; + txn.commit().await?; Ok(meta.config) } @@ -206,11 +187,7 @@ impl Backend { pub async fn delete_basin(&self, basin: BasinName) -> Result<(), DeleteBasinError> { let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; let meta_key = kv::basin_meta::ser_key(&basin); - let Some((mut meta, seq)) = db_txn_get_with(&txn, &meta_key, |entry| { - Ok((kv::basin_meta::deser_value(entry.value)?, entry.seq)) - }) - .await? - else { + let Some(mut meta) = db_txn_get(&txn, &meta_key, kv::basin_meta::deser_value).await? else { return Err(BasinNotFoundError { basin }.into()); }; if meta.deleted_at.is_none() { @@ -220,13 +197,9 @@ impl Backend { kv::basin_deletion_pending::ser_key(&basin), kv::basin_deletion_pending::ser_value(&StreamNameStartAfter::default()), )?; - db_txn_commit_durable(txn).await?; - } else { - // A retry may observe the marker before the first delete has flushed. - drop(txn); - self.await_durable_seq(seq).await?; + txn.commit().await?; + self.bgtask_trigger(BgtaskTrigger::BasinDeletion); } - self.bgtask_trigger(BgtaskTrigger::BasinDeletion); Ok(()) } } diff --git a/lite/src/backend/bgtasks/basin_deletion.rs b/lite/src/backend/bgtasks/basin_deletion.rs index 1dbb4344..e11563be 100644 --- a/lite/src/backend/bgtasks/basin_deletion.rs +++ b/lite/src/backend/bgtasks/basin_deletion.rs @@ -10,7 +10,6 @@ use slatedb::{ }; use tracing::instrument; -use super::{ItemProgress, PageProgress}; use crate::backend::{ Backend, error::{BasinDeletionError, ListStreamsError, StorageError}, @@ -26,17 +25,17 @@ impl Backend { if page.values.is_empty() { return Ok(page.has_more); } - let mut progress = PageProgress::new(page.has_more); let mut processed = stream::iter(page.values) .map(|(basin, cursor)| { let backend = self.clone(); async move { backend.process_basin_deletion(basin, cursor).await } }) .buffer_unordered(CONCURRENCY); + let mut has_more = page.has_more; while let Some(result) = processed.next().await { - progress.record(result?); + has_more |= result?; } - Ok(progress.should_continue()) + Ok(has_more) } async fn list_basin_deletion_pending( @@ -69,7 +68,7 @@ impl Backend { &self, basin: BasinName, cursor: StreamNameStartAfter, - ) -> Result { + ) -> Result { let request = ListStreamsRequest { prefix: StreamNamePrefix::default(), start_after: cursor.clone(), @@ -95,21 +94,20 @@ impl Backend { if page.has_more { self.set_basin_deletion_cursor(&basin, &last_stream.expect("non-empty stream page")) .await?; - Ok(ItemProgress::Advanced) + Ok(true) } else if last_stream.is_some() || !cursor.as_ref().is_empty() { // Streams still pending deletion or cursor was advanced past // earlier entries. Reset cursor so the next tick re-scans from - // the beginning. A reset is not progress: counting it would - // rescan a multi-page basin in a tight loop. + // the beginning. if !cursor.as_ref().is_empty() { self.set_basin_deletion_cursor(&basin, &StreamNameStartAfter::default()) .await?; } - Ok(ItemProgress::Blocked) + Ok(false) } else { // No streams from the very beginning — safe to complete. self.complete_basin_deletion(&basin).await?; - Ok(ItemProgress::Completed) + Ok(false) } } @@ -124,7 +122,7 @@ impl Backend { kv::basin_deletion_pending::ser_key(basin), kv::basin_deletion_pending::ser_value(cursor), ); - self.db.write(batch).await?.await_durable().await?; + self.db.write(batch).await?; Ok(()) } @@ -133,7 +131,7 @@ impl Backend { let mut batch = WriteBatch::new(); batch.delete(kv::basin_meta::ser_key(basin)); batch.delete(kv::basin_deletion_pending::ser_key(basin)); - self.db.write(batch).await?.await_durable().await?; + self.db.write(batch).await?; Ok(()) } } @@ -151,7 +149,7 @@ mod tests { use time::OffsetDateTime; use super::super::tests::test_backend; - use crate::backend::{Backend, kv, test_util::DbWriteTestExt as _}; + use crate::backend::{Backend, kv}; fn basin_meta(deleted_at: Option) -> kv::basin_meta::BasinMeta { kv::basin_meta::BasinMeta { @@ -187,16 +185,16 @@ mod tests { kv::basin_meta::ser_key(basin), kv::basin_meta::ser_value(&basin_meta(Some(OffsetDateTime::now_utc()))), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( kv::basin_deletion_pending::ser_key(basin), kv::basin_deletion_pending::ser_value(&StreamNameStartAfter::default()), ) - .assert_durable() - .await; + .await + .unwrap(); } async fn seed_tombstoned_streams(backend: &Backend, basin: &BasinName, count: usize) { @@ -209,7 +207,7 @@ mod tests { kv::stream_meta::ser_value(&stream_meta(Some(deleted_at))), ); } - backend.db.write(batch).assert_durable().await; + backend.db.write(batch).await.unwrap(); } #[tokio::test] @@ -223,16 +221,16 @@ mod tests { kv::basin_meta::ser_key(&basin), kv::basin_meta::ser_value(&basin_meta(Some(OffsetDateTime::now_utc()))), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( kv::basin_deletion_pending::ser_key(&basin), kv::basin_deletion_pending::ser_value(&StreamNameStartAfter::default()), ) - .assert_durable() - .await; + .await + .unwrap(); let has_more = backend.clone().tick_basin_deletion().await.unwrap(); assert!(!has_more); @@ -268,17 +266,8 @@ mod tests { kv::stream_meta::ser_key(&basin, &stream), kv::stream_meta::ser_value(&stream_meta(None)), ) - .assert_durable() - .await; - - backend - .db - .put( - kv::stream_id_mapping::ser_key(crate::stream_id::StreamId::new(&basin, &stream)), - kv::stream_id_mapping::ser_value(&basin, &stream), - ) - .assert_durable() - .await; + .await + .unwrap(); let has_more = backend.clone().tick_basin_deletion().await.unwrap(); assert!(!has_more); @@ -395,16 +384,16 @@ mod tests { kv::basin_meta::ser_key(&basin), kv::basin_meta::ser_value(&basin_meta(Some(OffsetDateTime::now_utc()))), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( kv::basin_deletion_pending::ser_key(&basin), kv::basin_deletion_pending::ser_value(&cursor), ) - .assert_durable() - .await; + .await + .unwrap(); // First tick resets cursor from past-end back to the beginning. let has_more = backend.clone().tick_basin_deletion().await.unwrap(); @@ -446,8 +435,8 @@ mod tests { kv::stream_meta::ser_key(&basin, &stream), kv::stream_meta::ser_value(&stream_meta(Some(deleted_at))), ) - .assert_durable() - .await; + .await + .unwrap(); let has_more = backend.clone().tick_basin_deletion().await.unwrap(); assert!(!has_more); @@ -485,8 +474,8 @@ mod tests { kv::stream_meta::ser_key(&basin, &stream), kv::stream_meta::ser_value(&stream_meta(Some(deleted_at))), ) - .assert_durable() - .await; + .await + .unwrap(); // First tick: blocked by tombstoned stream. let has_more = backend.clone().tick_basin_deletion().await.unwrap(); @@ -504,8 +493,8 @@ mod tests { backend .db .delete(kv::stream_meta::ser_key(&basin, &stream)) - .assert_durable() - .await; + .await + .unwrap(); // Second tick: no streams remain, completes. let has_more = backend.clone().tick_basin_deletion().await.unwrap(); diff --git a/lite/src/backend/bgtasks/mod.rs b/lite/src/backend/bgtasks/mod.rs index ed219387..a92916f8 100644 --- a/lite/src/backend/bgtasks/mod.rs +++ b/lite/src/backend/bgtasks/mod.rs @@ -9,63 +9,6 @@ mod basin_deletion; mod stream_doe; mod stream_trim; -/// Keep draining the backlog while at least one item makes progress. A page where -/// every item is blocked waits for the next tick instead of retrying in a tight loop. -struct PageProgress { - has_more: bool, - any_progressed: bool, -} - -enum ItemProgress { - /// Made progress, and more work for this item can run immediately. - Advanced, - Completed, - /// Made no progress; the item waits for a later tick. - Blocked, -} - -impl PageProgress { - fn new(has_more: bool) -> Self { - Self { - has_more, - any_progressed: false, - } - } - - fn record(&mut self, item: ItemProgress) { - match item { - ItemProgress::Advanced => { - self.has_more = true; - self.any_progressed = true; - } - ItemProgress::Completed => self.any_progressed = true, - ItemProgress::Blocked => {} - } - } - - fn record_result( - &mut self, - result: Result, - is_conflict: fn(&E) -> bool, - ) -> Result, E> { - match result { - Ok(value) => { - self.record(ItemProgress::Completed); - Ok(Some(value)) - } - Err(err) if is_conflict(&err) => { - self.record(ItemProgress::Blocked); - Ok(None) - } - Err(err) => Err(err), - } - } - - fn should_continue(&self) -> bool { - self.has_more && self.any_progressed - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum BgtaskTrigger { BasinDeletion, @@ -229,13 +172,4 @@ mod tests { assert_eq!(calls.load(Ordering::SeqCst), 3); } - - #[test] - fn blocked_page_waits_for_next_tick() { - let mut progress = PageProgress::new(true); - progress.record(ItemProgress::Blocked); - assert!(!progress.should_continue()); - progress.record(ItemProgress::Completed); - assert!(progress.should_continue()); - } } diff --git a/lite/src/backend/bgtasks/stream_doe.rs b/lite/src/backend/bgtasks/stream_doe.rs index d5c3a19d..d09b108d 100644 --- a/lite/src/backend/bgtasks/stream_doe.rs +++ b/lite/src/backend/bgtasks/stream_doe.rs @@ -1,21 +1,21 @@ -use bytes::Bytes; +use std::time::Duration; + use futures::{StreamExt, stream}; use indexmap::IndexMap; -use s2_common::{basin::BasinName, resources::Page, stream::StreamName}; +use itertools::Itertools; +use s2_common::resources::Page; use slatedb::{ - IsolationLevel, WriteBatch, + WriteBatch, config::{DurabilityLevel, ScanOptions}, }; +use tracing::instrument; -use super::PageProgress; use crate::{ backend::{ - Backend, doe, + Backend, error::{DeleteStreamError, StorageError, StreamDeleteOnEmptyError}, - kv, - store::{db_snapshot_get_with, db_txn_commit_durable, db_txn_get, db_txn_get_with}, - streamer::{TerminalTrimCondition, TerminalTrimOutcome}, - timestamp::TimestampSecs, + kv::{self, timestamp::TimestampSecs}, + streamer::{TerminalTrimCondition, doe_arm_delay}, }, stream_id::StreamId, }; @@ -23,1222 +23,780 @@ use crate::{ const PENDING_LIST_LIMIT: usize = 10_000; const CONCURRENCY: usize = 4; -#[derive(Clone, Copy, Debug)] -struct PendingCheck { - stream_id: StreamId, - check: kv::stream_doe_state::Check, +#[derive(Debug)] +struct PendingDoeBatch { + entries: Vec, + last_write_cutoff: Option, } -struct CheckSnapshot { - revision: u64, - creation_seq: u64, - config_seq: u64, - basin: BasinName, - stream: StreamName, +impl PendingDoeBatch { + fn new(entries: Vec) -> Self { + let last_write_cutoff = entries + .iter() + .filter_map(|entry| entry.last_write_cutoff()) + .max(); + Self { + entries, + last_write_cutoff, + } + } + + fn entries(&self) -> &[kv::stream_doe_deadline::Entry] { + &self.entries + } } impl Backend { pub(super) async fn tick_stream_doe(self) -> Result { - // Drain legacy keys regardless of their old deadlines. Their timestamps - // and values have no role in the new scheduler or deletion eligibility. - let mut progress = self.migrate_stream_doe().await?; - let page = self.list_pending_stream_doe(TimestampSecs::now()).await?; - progress.has_more |= page.has_more; + let now = TimestampSecs::now(); + let page = self.list_pending_stream_doe(now).await?; + if page.values.is_empty() { + return Ok(page.has_more); + } let mut processed = stream::iter(page.values) - .map(|pending| { + .map(|(stream_id, pending)| { let backend = self.clone(); - async move { backend.process_stream_doe(pending).await } + async move { backend.process_stream_doe(stream_id, pending).await } }) .buffer_unordered(CONCURRENCY); while let Some(result) = processed.next().await { - progress.record_result(result, StreamDeleteOnEmptyError::is_transaction_conflict)?; + result?; } - Ok(progress.should_continue()) + Ok(page.has_more) } async fn list_pending_stream_doe( &self, now: TimestampSecs, - ) -> Result, StorageError> { + ) -> Result, StorageError> { let scan_opts = ScanOptions { durability_filter: DurabilityLevel::Remote, ..Default::default() }; let mut it = self .db - .scan_with_options(kv::stream_doe_check::due_key_range(now), &scan_opts) + .scan_with_options(kv::stream_doe_deadline::expired_key_range(now), &scan_opts) .await?; - let mut pending = Vec::new(); + let mut pending: IndexMap> = IndexMap::new(); + let mut has_more = false; + let mut count = 0; while let Some(kv) = it.next().await? { - let (stream_id, check) = kv::stream_doe_check::deser_key(kv.key)?; - pending.push(PendingCheck { stream_id, check }); - if pending.len() == PENDING_LIST_LIMIT { - return Ok(Page::new(pending, true)); + let (deadline, stream_id) = kv::stream_doe_deadline::deser_key(kv.key)?; + let min_age = kv::stream_doe_deadline::deser_value(kv.value)?; + assert!(deadline <= now); + pending + .entry(stream_id) + .or_default() + .push(kv::stream_doe_deadline::Entry { deadline, min_age }); + count += 1; + if count == PENDING_LIST_LIMIT { + has_more = true; + break; } } - Ok(Page::new(pending, false)) + Ok(Page::new( + pending + .into_iter() + .map(|(stream_id, entries)| (stream_id, PendingDoeBatch::new(entries))) + .collect_vec(), + has_more, + )) } async fn process_stream_doe( &self, - pending: PendingCheck, + stream_id: StreamId, + pending: PendingDoeBatch, ) -> Result<(), StreamDeleteOnEmptyError> { - let Some(snapshot) = self.observe_doe_check(pending).await? else { - return Ok(()); - }; - let outcome = match self - .delete_stream_with_condition( - snapshot.basin.clone(), - snapshot.stream.clone(), - TerminalTrimCondition::DeleteOnEmpty { - expected_stream_creation_seq: snapshot.creation_seq, - expected_config_seq: snapshot.config_seq, - }, - ) - .await + if let Some(last_write_cutoff) = pending.last_write_cutoff + && let Some((basin, stream)) = self.stream_id_mapping(stream_id).await? { - Ok(outcome) => outcome, - Err(DeleteStreamError::StreamNotFound(_)) => TerminalTrimOutcome::Obsolete, - Err(err) => return Err(err.into()), - }; - self.finish_doe_check(pending, snapshot, outcome).await?; + match self + .delete_stream_with_condition( + basin, + stream, + TerminalTrimCondition::DeleteOnEmpty { last_write_cutoff }, + ) + .await + { + Ok(()) | Err(DeleteStreamError::StreamNotFound(_)) => {} + Err(err) => return Err(err.into()), + } + } + self.clear_doe_deadlines(stream_id, pending.entries()) + .await?; Ok(()) } - /// Capture the scheduler revision before the streamer's asynchronous scan. - async fn observe_doe_check( + #[instrument(ret, err, skip(self, pending), fields(num_deadlines = pending.len()))] + async fn clear_doe_deadlines( &self, - pending: PendingCheck, - ) -> Result, StorageError> { - // Observation needs a consistent view, but no transaction read set. The - // completion transaction validates this revision after the actor's scan. - let snapshot = self.db.snapshot().await?; - let state = db_snapshot_get_with( - &snapshot, - kv::stream_doe_state::ser_key(pending.stream_id), - |entry| Ok((kv::stream_doe_state::deser_value(entry.value)?, entry.seq)), - ) - .await?; - let Some((kv::stream_doe_state::State::Scheduled(_), revision)) = state - .filter(|(state, _)| *state == kv::stream_doe_state::State::Scheduled(pending.check)) - else { - drop(snapshot); - self.discard_doe_check(pending, state).await?; - return Ok(None); + stream_id: StreamId, + pending: &[kv::stream_doe_deadline::Entry], + ) -> Result<(), StorageError> { + let mut batch = WriteBatch::new(); + for entry in pending { + batch.delete(kv::stream_doe_deadline::ser_key(entry.deadline, stream_id)); + } + self.db.write(batch).await?; + Ok(()) + } + + pub(super) async fn arm_doe_on_full_trim( + &self, + stream_id: StreamId, + ) -> Result<(), StorageError> { + let Some((basin, stream)) = self.stream_id_mapping(stream_id).await? else { + return Ok(()); }; - let mapping = db_snapshot_get_with( - &snapshot, - kv::stream_id_mapping::ser_key(pending.stream_id), - |entry| Ok((kv::stream_id_mapping::deser_value(entry.value)?, entry.seq)), - ) - .await?; - if let Some(((basin, stream), creation_seq)) = mapping - && revision >= creation_seq - && let Some((meta, config_seq)) = db_snapshot_get_with( - &snapshot, - kv::stream_meta::ser_key(&basin, &stream), - |entry| Ok((kv::stream_meta::deser_value(entry.value)?, entry.seq)), + let Some(meta) = self + .db_get( + &kv::stream_meta::ser_key(&basin, &stream), + kv::stream_meta::deser_value, ) .await? - && meta.deleted_at.is_none() - && meta.config.delete_on_empty.min_age().is_some() - { - return Ok(Some(CheckSnapshot { - revision, - creation_seq, - config_seq, - basin, - stream, - })); - } - drop(snapshot); - self.discard_doe_check(pending, state).await?; - Ok(None) - } - - /// Obsolete work needs a transaction only when it is actually discarded. - /// Revalidate the snapshot so cleanup cannot erase a concurrent wake. - async fn discard_doe_check( - &self, - pending: PendingCheck, - observed: Option<(kv::stream_doe_state::State, u64)>, - ) -> Result<(), StorageError> { - let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; - let current = db_txn_get_with( - &txn, - kv::stream_doe_state::ser_key(pending.stream_id), - |entry| Ok((kv::stream_doe_state::deser_value(entry.value)?, entry.seq)), - ) - .await?; - if current != observed { + else { + return Ok(()); + }; + if meta.deleted_at.is_some() { return Ok(()); } - match current { - Some((kv::stream_doe_state::State::Scheduled(check), _)) if check == pending.check => { - doe::replace( - &txn, - pending.stream_id, - Some(kv::stream_doe_state::State::Scheduled(check)), - None, - )?; - } - _ => { - txn.delete(kv::stream_doe_check::ser_key( - pending.stream_id, - pending.check, - ))?; - } - } - db_txn_commit_durable(txn).await?; - Ok(()) - } - - async fn finish_doe_check( - &self, - pending: PendingCheck, - snapshot: CheckSnapshot, - outcome: TerminalTrimOutcome, - ) -> Result<(), StorageError> { - let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; - let state = db_txn_get_with( - &txn, - kv::stream_doe_state::ser_key(pending.stream_id), - |entry| Ok((kv::stream_doe_state::deser_value(entry.value)?, entry.seq)), - ) - .await?; - if state - != Some(( - kv::stream_doe_state::State::Scheduled(pending.check), - snapshot.revision, - )) - { - // A trim/configuration event owns the next check, even if it kept the - // same ticket. Consuming or postponing it here would lose that wake. + let Some(min_age) = meta.config.delete_on_empty.min_age() else { return Ok(()); - } - // Successful deletion marks metadata and advances its sequence. Deferred - // outcomes must still belong to the configuration that was inspected. - if outcome != TerminalTrimOutcome::DeletionPending { - let config_seq = db_txn_get_with( - &txn, - kv::stream_meta::ser_key(&snapshot.basin, &snapshot.stream), - |entry| Ok(entry.seq), + }; + let deadline = TimestampSecs::after(doe_arm_delay(Duration::ZERO, min_age)); + self.db + .put( + kv::stream_doe_deadline::ser_key(deadline, stream_id), + kv::stream_doe_deadline::ser_value(min_age), ) .await?; - if config_seq != Some(snapshot.config_seq) { - return Ok(()); - } - } - let next = match outcome { - TerminalTrimOutcome::RetryAt(at) => Some(kv::stream_doe_state::State::Scheduled( - kv::stream_doe_state::Check { - at, - id: rand::random(), - }, - )), - TerminalTrimOutcome::Parked => Some(kv::stream_doe_state::State::Parked), - TerminalTrimOutcome::DeletionPending | TerminalTrimOutcome::Obsolete => None, - }; - // Consuming the old check and installing its successor is one durable - // transaction, including when the caller disappears during commit. - doe::replace( - &txn, - pending.stream_id, - Some(kv::stream_doe_state::State::Scheduled(pending.check)), - next, - )?; - db_txn_commit_durable(txn).await?; Ok(()) } +} - async fn migrate_stream_doe(&self) -> Result { +#[cfg(test)] +mod tests { + use std::{str::FromStr, time::Duration}; + + use s2_common::{ + basin::BasinName, + config::{ + BasinConfig, DeleteOnEmptyReconfiguration, OptionalStreamConfig, RetentionPolicy, + StreamReconfiguration, + }, + maybe::Maybe, + record::StreamPosition, + stream::StreamName, + }; + use slatedb::config::{DurabilityLevel, ScanOptions}; + use time::OffsetDateTime; + + use super::{super::tests::test_backend, PendingDoeBatch, TimestampSecs}; + use crate::{ + backend::{Backend, kv}, + stream_id::StreamId, + }; + + const MIN_AGE: Duration = Duration::from_secs(60); + + fn stream_meta_with_config( + config: OptionalStreamConfig, + created_at: OffsetDateTime, + ) -> kv::stream_meta::StreamMeta { + kv::stream_meta::StreamMeta { + config: config.into(), + cipher: None, + created_at, + deleted_at: None, + creation_idempotency_key: None, + } + } + + fn stream_meta_with_doe_min_age(min_age: Duration) -> kv::stream_meta::StreamMeta { + let mut config = OptionalStreamConfig::default(); + config.delete_on_empty.min_age = Some(min_age); + stream_meta_with_config(config, OffsetDateTime::now_utc()) + } + + async fn seed_stream_with_meta( + backend: &Backend, + basin: &BasinName, + stream: &StreamName, + meta: kv::stream_meta::StreamMeta, + ) -> StreamId { + let stream_id = StreamId::new(basin, stream); + backend + .db + .put( + kv::basin_meta::ser_key(basin), + kv::basin_meta::ser_value(&kv::basin_meta::BasinMeta { + config: BasinConfig::default(), + created_at: OffsetDateTime::now_utc(), + deleted_at: None, + creation_idempotency_key: None, + }), + ) + .await + .unwrap(); + backend + .db + .put( + kv::stream_meta::ser_key(basin, stream), + kv::stream_meta::ser_value(&meta), + ) + .await + .unwrap(); + backend + .db + .put( + kv::stream_id_mapping::ser_key(stream_id), + kv::stream_id_mapping::ser_value(basin, stream), + ) + .await + .unwrap(); + stream_id + } + + async fn list_doe_entries(backend: &Backend) -> Vec<(TimestampSecs, StreamId, Duration)> { let scan_opts = ScanOptions { durability_filter: DurabilityLevel::Remote, ..Default::default() }; - let mut it = self + let mut it = backend .db .scan_with_options( kv::key_type_range(kv::KeyType::StreamDeleteOnEmptyDeadline), &scan_opts, ) - .await?; - let mut pending: IndexMap> = IndexMap::new(); - let mut count = 0; - while let Some(entry) = it.next().await? { - let (_, stream_id, _) = kv::stream_doe_deadline::deser_key(entry.key.clone())?; - pending.entry(stream_id).or_default().push(entry.key); - count += 1; - if count == PENDING_LIST_LIMIT { - break; - } + .await + .unwrap(); + let mut entries = Vec::new(); + while let Some(kv) = it.next().await.unwrap() { + let (deadline, stream_id) = kv::stream_doe_deadline::deser_key(kv.key).unwrap(); + let min_age = kv::stream_doe_deadline::deser_value(kv.value).unwrap(); + entries.push((deadline, stream_id, min_age)); } - let mut progress = PageProgress::new(count == PENDING_LIST_LIMIT); - if pending.is_empty() { - return Ok(progress); - } - let snapshot = self.db.snapshot().await?; - let mut migrations = stream::iter(pending) - .map(|(stream_id, keys)| self.migrate_doe_deadlines(&snapshot, stream_id, keys)) - .buffer_unordered(CONCURRENCY); - let mut cleanup = WriteBatch::new(); - while let Some(result) = migrations.next().await { - if let Some(keys) = - progress.record_result(result, StorageError::is_transaction_conflict)? - { - for key in keys { - cleanup.delete(key); - } - } - } - if !cleanup.is_empty() { - // Later enablement/recreation installs its own schedule. Only - // legacy keys are removed; new state and its revision stay intact. - // Durability also covers any commits observed by the snapshot. - self.db.write(cleanup).await?.await_durable().await?; - } - Ok(progress) + entries } - async fn migrate_doe_deadlines( - &self, - snapshot: &slatedb::DbSnapshot, + async fn put_tail_position( + backend: &Backend, stream_id: StreamId, - keys: Vec, - ) -> Result, StorageError> { - if needs_doe_migration(snapshot, stream_id).await? { - // Initialization and removal remain atomic. Recheck all eligibility - // in the transaction after the snapshot. - self.initialize_doe_from_deadlines(stream_id, keys).await?; - Ok(Vec::new()) - } else { - Ok(keys) - } + position: StreamPosition, + ) -> TimestampSecs { + let key = kv::stream_tail_position::ser_key(stream_id); + backend + .db + .put(key.clone(), kv::stream_tail_position::ser_value(position)) + .await + .unwrap(); + let kv = backend + .db + .get_key_value(key) + .await + .unwrap() + .expect("tail position should exist"); + TimestampSecs::from_millis(kv.create_ts) } - async fn initialize_doe_from_deadlines( - &self, + fn deadline_after(write_timestamp: TimestampSecs, age: Duration) -> TimestampSecs { + let deadline_secs = u64::from(write_timestamp.as_u32()) + .saturating_add(age.as_secs()) + .min(u64::from(u32::MAX)) as u32; + TimestampSecs::from_secs(deadline_secs) + } + + async fn process_pending_stream_doe_at( + backend: &Backend, stream_id: StreamId, - keys: Vec, - ) -> Result<(), StorageError> { - let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; - let mapping = db_txn_get_with(&txn, kv::stream_id_mapping::ser_key(stream_id), |entry| { - Ok((kv::stream_id_mapping::deser_value(entry.value)?, entry.seq)) - }) - .await?; - if let Some(((basin, stream), creation_seq)) = mapping - && let Some(meta) = db_txn_get( - &txn, - kv::stream_meta::ser_key(&basin, &stream), - kv::stream_meta::deser_value, + now: TimestampSecs, + ) { + let mut page = backend.list_pending_stream_doe(now).await.unwrap(); + assert!(!page.has_more); + assert_eq!(page.values.len(), 1); + let (pending_stream_id, pending) = page.values.pop().unwrap(); + assert_eq!(pending_stream_id, stream_id); + + backend + .process_stream_doe(stream_id, pending) + .await + .unwrap(); + } + + #[tokio::test] + async fn stream_doe_marks_deleted_and_clears_deadline() { + let backend = test_backend().await; + let basin = BasinName::from_str("doe-basin").unwrap(); + let stream = StreamName::from_str("doe-stream").unwrap(); + let stream_id = seed_stream_with_meta( + &backend, + &basin, + &stream, + stream_meta_with_doe_min_age(MIN_AGE), + ) + .await; + let write_timestamp = put_tail_position( + &backend, + stream_id, + StreamPosition { + seq_num: 1, + timestamp: 1234, + }, + ) + .await; + let deadline = deadline_after(write_timestamp, MIN_AGE); + + backend + .db + .put( + kv::stream_doe_deadline::ser_key(deadline, stream_id), + kv::stream_doe_deadline::ser_value(MIN_AGE), ) - .await? - && meta.deleted_at.is_none() - && meta.config.delete_on_empty.min_age().is_some() - { - let state = db_txn_get_with(&txn, kv::stream_doe_state::ser_key(stream_id), |entry| { - Ok((kv::stream_doe_state::deser_value(entry.value)?, entry.seq)) - }) - .await?; - if state.is_none_or(|(_, seq)| seq < creation_seq) { - let next = kv::stream_doe_state::State::Scheduled(kv::stream_doe_state::Check { - at: TimestampSecs::now(), - id: rand::random(), - }); - doe::replace(&txn, stream_id, state.map(|(state, _)| state), Some(next))?; - } - // Existing state, including Parked, is deliberately untouched. - } - for key in keys { - txn.delete(key)?; - } - db_txn_commit_durable(txn).await?; - Ok(()) + .await + .unwrap(); + + process_pending_stream_doe_at(&backend, stream_id, deadline).await; + + let meta = backend + .db + .get(kv::stream_meta::ser_key(&basin, &stream)) + .await + .unwrap() + .expect("stream meta should remain"); + let decoded = kv::stream_meta::deser_value(meta).unwrap(); + assert!(decoded.deleted_at.is_some()); + + let deadline_key = backend + .db + .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) + .await + .unwrap(); + assert!(deadline_key.is_none()); } -} -async fn needs_doe_migration( - snapshot: &slatedb::DbSnapshot, - stream_id: StreamId, -) -> Result { - let Some(((basin, stream), creation_seq)) = db_snapshot_get_with( - snapshot, - kv::stream_id_mapping::ser_key(stream_id), - |entry| Ok((kv::stream_id_mapping::deser_value(entry.value)?, entry.seq)), - ) - .await? - else { - return Ok(false); - }; - let Some(meta) = db_snapshot_get_with( - snapshot, - kv::stream_meta::ser_key(&basin, &stream), - |entry| kv::stream_meta::deser_value(entry.value), - ) - .await? - else { - return Ok(false); - }; - if meta.deleted_at.is_some() || meta.config.delete_on_empty.min_age().is_none() { - return Ok(false); + #[tokio::test] + async fn stream_doe_deletes_never_written_stream() { + let backend = test_backend().await; + let basin = BasinName::from_str("doe-basin-never").unwrap(); + let stream = StreamName::from_str("doe-stream-never").unwrap(); + let stream_id = StreamId::new(&basin, &stream); + let min_age = MIN_AGE; + let meta = stream_meta_with_doe_min_age(min_age); + + seed_stream_with_meta(&backend, &basin, &stream, meta).await; + + let write_timestamp = put_tail_position(&backend, stream_id, StreamPosition::MIN).await; + let deadline = deadline_after(write_timestamp, min_age); + backend + .db + .put( + kv::stream_doe_deadline::ser_key(deadline, stream_id), + kv::stream_doe_deadline::ser_value(min_age), + ) + .await + .unwrap(); + + process_pending_stream_doe_at(&backend, stream_id, deadline).await; + + let meta = backend + .db + .get(kv::stream_meta::ser_key(&basin, &stream)) + .await + .unwrap() + .expect("stream meta should remain"); + let decoded = kv::stream_meta::deser_value(meta).unwrap(); + assert!(decoded.deleted_at.is_some()); + + let deadline_key = backend + .db + .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) + .await + .unwrap(); + assert!(deadline_key.is_none()); } - let state_seq = db_snapshot_get_with( - snapshot, - kv::stream_doe_state::ser_key(stream_id), - |entry| { - kv::stream_doe_state::deser_value(entry.value)?; - Ok(entry.seq) - }, - ) - .await?; - Ok(state_seq.is_none_or(|seq| seq < creation_seq)) -} -#[cfg(test)] -mod tests { - use std::time::Duration; + #[tokio::test] + async fn stream_doe_skips_recent_tail_write() { + let backend = test_backend().await; + let basin = BasinName::from_str("doe-basin-recent").unwrap(); + let stream = StreamName::from_str("doe-stream-recent").unwrap(); + let stream_id = seed_stream_with_meta( + &backend, + &basin, + &stream, + stream_meta_with_doe_min_age(MIN_AGE), + ) + .await; + let write_timestamp = put_tail_position( + &backend, + stream_id, + StreamPosition { + seq_num: 1, + timestamp: 1234, + }, + ) + .await; + let deadline = write_timestamp; - use s2_common::{ - config::{ - DeleteOnEmptyReconfiguration, OptionalStreamConfig, RetentionPolicy, - StreamReconfiguration, - }, - maybe::Maybe, - record::{CommandRecord, MeteredExt as _, Record}, - resources::ProvisionMode, - stream::{AppendInput, AppendRecord, AppendRecordParts}, - }; + backend + .db + .put( + kv::stream_doe_deadline::ser_key(deadline, stream_id), + kv::stream_doe_deadline::ser_value(MIN_AGE), + ) + .await + .unwrap(); - use super::*; - use crate::backend::{ - bgtasks::{run_tick, tests::test_backend}, - test_util::{DbWriteTestExt as _, create_stream}, - }; + process_pending_stream_doe_at(&backend, stream_id, deadline).await; - fn config(min_age: u64, retention: RetentionPolicy) -> OptionalStreamConfig { - let mut config = OptionalStreamConfig { - retention_policy: Some(retention), + let meta = backend + .db + .get(kv::stream_meta::ser_key(&basin, &stream)) + .await + .unwrap() + .expect("stream meta should remain"); + let decoded = kv::stream_meta::deser_value(meta).unwrap(); + assert!(decoded.deleted_at.is_none()); + + let deadline_key = backend + .db + .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) + .await + .unwrap(); + assert!(deadline_key.is_none()); + } + + #[tokio::test] + async fn stream_doe_skips_stream_with_records() { + let backend = test_backend().await; + let basin = BasinName::from_str("doe-basin-nonempty").unwrap(); + let stream = StreamName::from_str("doe-stream-nonempty").unwrap(); + let stream_id = seed_stream_with_meta( + &backend, + &basin, + &stream, + stream_meta_with_doe_min_age(MIN_AGE), + ) + .await; + let deadline = TimestampSecs::now(); + + let pos = StreamPosition { + seq_num: 1, + timestamp: 1234, + }; + backend + .db + .put( + kv::stream_record_timestamp::ser_key(stream_id, pos), + kv::stream_record_timestamp::ser_value(), + ) + .await + .unwrap(); + backend + .db + .put( + kv::stream_doe_deadline::ser_key(deadline, stream_id), + kv::stream_doe_deadline::ser_value(MIN_AGE), + ) + .await + .unwrap(); + + let has_more = backend.clone().tick_stream_doe().await.unwrap(); + assert!(!has_more); + + let meta = backend + .db + .get(kv::stream_meta::ser_key(&basin, &stream)) + .await + .unwrap() + .expect("stream meta should remain"); + let decoded = kv::stream_meta::deser_value(meta).unwrap(); + assert!(decoded.deleted_at.is_none()); + + let deadline_key = backend + .db + .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) + .await + .unwrap(); + assert!(deadline_key.is_none()); + + let timestamp_key = backend + .db + .get(kv::stream_record_timestamp::ser_key(stream_id, pos)) + .await + .unwrap(); + assert!(timestamp_key.is_some()); + } + + #[tokio::test] + async fn stream_doe_ignores_future_deadline() { + let backend = test_backend().await; + let basin = BasinName::from_str("doe-basin-future").unwrap(); + let stream = StreamName::from_str("doe-stream-future").unwrap(); + let stream_id = seed_stream_with_meta( + &backend, + &basin, + &stream, + stream_meta_with_doe_min_age(MIN_AGE), + ) + .await; + let deadline = TimestampSecs::after(Duration::from_secs(3600)); + + backend + .db + .put( + kv::stream_doe_deadline::ser_key(deadline, stream_id), + kv::stream_doe_deadline::ser_value(MIN_AGE), + ) + .await + .unwrap(); + + let has_more = backend.clone().tick_stream_doe().await.unwrap(); + assert!(!has_more); + + let meta = backend + .db + .get(kv::stream_meta::ser_key(&basin, &stream)) + .await + .unwrap() + .expect("stream meta should remain"); + let decoded = kv::stream_meta::deser_value(meta).unwrap(); + assert!(decoded.deleted_at.is_none()); + + let deadline_key = backend + .db + .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) + .await + .unwrap(); + assert!(deadline_key.is_some()); + } + + #[tokio::test] + async fn stream_doe_groups_multiple_deadlines() { + let backend = test_backend().await; + let basin = BasinName::from_str("doe-basin-multi").unwrap(); + let stream = StreamName::from_str("doe-stream-multi").unwrap(); + let stream_id = seed_stream_with_meta( + &backend, + &basin, + &stream, + stream_meta_with_doe_min_age(MIN_AGE), + ) + .await; + let far_future = TimestampSecs::after(Duration::from_secs(3600)); + let deadline_a = TimestampSecs::after(Duration::ZERO); + let deadline_b = TimestampSecs::after(Duration::from_secs(1)); + + backend + .db + .put( + kv::stream_doe_deadline::ser_key(deadline_a, stream_id), + kv::stream_doe_deadline::ser_value(MIN_AGE), + ) + .await + .unwrap(); + backend + .db + .put( + kv::stream_doe_deadline::ser_key(deadline_b, stream_id), + kv::stream_doe_deadline::ser_value(MIN_AGE), + ) + .await + .unwrap(); + + let page = backend.list_pending_stream_doe(far_future).await.unwrap(); + assert!(!page.has_more); + assert_eq!(page.values.len(), 1); + let (pending_stream_id, pending) = page.values.into_iter().next().unwrap(); + assert_eq!(pending_stream_id, stream_id); + let mut deadlines: Vec<_> = pending + .entries() + .iter() + .map(|entry| entry.deadline) + .collect(); + deadlines.sort(); + let mut expected = vec![deadline_a, deadline_b]; + expected.sort(); + assert_eq!(deadlines, expected); + + backend + .process_stream_doe(stream_id, pending) + .await + .unwrap(); + + for deadline in [deadline_a, deadline_b] { + let deadline_key = backend + .db + .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) + .await + .unwrap(); + assert!(deadline_key.is_none()); + } + } + + #[tokio::test] + async fn stream_doe_uses_latest_eligible_cutoff_across_pending_entries() { + let backend = test_backend().await; + let basin = BasinName::from_str("doe-basin-pairs").unwrap(); + let stream = StreamName::from_str("doe-stream-pairs").unwrap(); + let stream_id = seed_stream_with_meta( + &backend, + &basin, + &stream, + stream_meta_with_doe_min_age(MIN_AGE), + ) + .await; + + let write_timestamp = put_tail_position( + &backend, + stream_id, + StreamPosition { + seq_num: 1, + timestamp: 1234, + }, + ) + .await; + + backend + .process_stream_doe( + stream_id, + PendingDoeBatch::new(vec![ + kv::stream_doe_deadline::Entry { + deadline: deadline_after(write_timestamp, Duration::from_secs(50)), + min_age: Duration::from_secs(100), + }, + kv::stream_doe_deadline::Entry { + deadline: deadline_after(write_timestamp, Duration::from_secs(100)), + min_age: Duration::from_secs(10), + }, + ]), + ) + .await + .unwrap(); + + let meta = backend + .db + .get(kv::stream_meta::ser_key(&basin, &stream)) + .await + .unwrap() + .expect("stream meta should remain"); + let decoded = kv::stream_meta::deser_value(meta).unwrap(); + assert!(decoded.deleted_at.is_some()); + } + + #[tokio::test] + async fn reconfigure_enabling_doe_on_nonempty_retained_stream_arms_future_deadline() { + let backend = test_backend().await; + let basin = BasinName::from_str("doe-basin-enable").unwrap(); + let stream = StreamName::from_str("doe-stream-enable").unwrap(); + let config = OptionalStreamConfig { + retention_policy: Some(RetentionPolicy::Age(Duration::from_secs(120))), ..Default::default() }; - config.delete_on_empty.min_age = Some(Duration::from_secs(min_age)); - config - } - - async fn state( - backend: &Backend, - stream_id: StreamId, - ) -> Option<(kv::stream_doe_state::State, u64)> { - backend - .db_get_with(kv::stream_doe_state::ser_key(stream_id), |entry| { - Ok((kv::stream_doe_state::deser_value(entry.value)?, entry.seq)) - }) - .await - .unwrap() - } - - async fn scheduled(backend: &Backend, stream_id: StreamId) -> kv::stream_doe_state::Check { - let Some((kv::stream_doe_state::State::Scheduled(check), _)) = - state(backend, stream_id).await - else { - panic!("expected a scheduled check"); + let stream_id = seed_stream_with_meta( + &backend, + &basin, + &stream, + stream_meta_with_config(config, OffsetDateTime::now_utc()), + ) + .await; + let pos = StreamPosition { + seq_num: 1, + timestamp: 1234, }; - let checks = backend - .list_pending_stream_doe(TimestampSecs::MAX) - .await - .unwrap(); - assert_eq!(checks.values.len(), 1); - assert_eq!(checks.values[0].stream_id, stream_id); - assert_eq!(checks.values[0].check, check); - check - } - - async fn due(backend: &Backend, stream_id: StreamId) -> PendingCheck { - let txn = backend - .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - doe::schedule(&txn, stream_id, TimestampSecs::ZERO) - .await - .unwrap(); - db_txn_commit_durable(txn).await.unwrap(); - PendingCheck { - stream_id, - check: scheduled(backend, stream_id).await, - } - } - - async fn append(backend: &Backend, basin: &BasinName, stream: &StreamName, record: Record) { - let record: AppendRecord = AppendRecordParts { - timestamp: None, - record: record.metered(), - } - .try_into() - .unwrap(); backend - .open_for_append(basin, stream, None, OptionalStreamConfig::default()) - .await - .unwrap() - .append(AppendInput { - records: vec![record].try_into().unwrap(), - match_seq_num: None, - fencing_token: None, - }) + .db + .put( + kv::stream_tail_position::ser_key(stream_id), + kv::stream_tail_position::ser_value(pos), + ) + .await + .unwrap(); + backend + .db + .put( + kv::stream_record_timestamp::ser_key(stream_id, pos), + kv::stream_record_timestamp::ser_value(), + ) .await .unwrap(); - } - fn record() -> Record { - Record::try_from_parts(vec![], Bytes::from_static(b"live")).unwrap() - } - - async fn configure_min_age( - backend: &Backend, - basin: &BasinName, - stream: &StreamName, - min_age: u64, - via_ensure: bool, - ) { - let min_age = Duration::from_secs(min_age); - if via_ensure { - let mut config = backend - .get_stream_config(basin.clone(), stream.clone()) - .await - .unwrap(); - config.delete_on_empty.min_age = min_age; - backend - .provision_stream( - basin.clone(), - stream.clone(), - config.into(), - ProvisionMode::Ensure, - ) - .await - .unwrap(); - } else { - backend - .reconfigure_stream( - basin.clone(), - stream.clone(), - StreamReconfiguration { - delete_on_empty: Maybe::from(Some(DeleteOnEmptyReconfiguration { - min_age: Maybe::from(Some(min_age)), - })), - ..Default::default() - }, - ) - .await - .unwrap(); - } - } - - async fn configure_retention( - backend: &Backend, - basin: &BasinName, - stream: &StreamName, - age: u64, - ) { + let min_age = Duration::from_secs(30); + let expected_delay = + crate::backend::streamer::doe_arm_delay(Duration::from_secs(120), min_age); + let lower_bound = TimestampSecs::now(); backend .reconfigure_stream( - basin.clone(), - stream.clone(), + basin, + stream, StreamReconfiguration { - retention_policy: Maybe::from(Some(RetentionPolicy::Age(Duration::from_secs( - age, - )))), + delete_on_empty: Maybe::from(Some(DeleteOnEmptyReconfiguration { + min_age: Maybe::from(Some(min_age)), + })), ..Default::default() }, ) .await .unwrap(); - } + let upper_bound = TimestampSecs::now(); - async fn legacy(backend: &Backend, stream_id: StreamId, id: Option) -> Bytes { - let key = kv::stream_doe_deadline::ser_key(TimestampSecs::MAX, stream_id, id); - // Migration must not interpret the old min_age, even for future keys. - backend.db.put(&key, [0xff]).assert_durable().await; - key + let entries = list_doe_entries(&backend).await; + assert_eq!(entries.len(), 1); + let (deadline, scheduled_stream_id, scheduled_min_age) = entries[0]; + assert_eq!(scheduled_stream_id, stream_id); + assert_eq!(scheduled_min_age, min_age); + + let lower_secs = u64::from(lower_bound.as_u32()).saturating_add(expected_delay.as_secs()); + let upper_secs = u64::from(upper_bound.as_u32()).saturating_add(expected_delay.as_secs()); + let deadline_secs = u64::from(deadline.as_u32()); + assert!(lower_secs <= deadline_secs); + assert!(deadline_secs <= upper_secs); } #[tokio::test] - async fn creation_schedules_once_and_appends_do_not_write_schedules() { + async fn reconfigure_changing_enabled_doe_does_not_arm_new_deadline() { let backend = test_backend().await; - let before = TimestampSecs::after(Duration::from_secs(60)); - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - let check = scheduled(&backend, stream_id).await; - assert!(check.at >= before); - assert!(check.at <= TimestampSecs::after(Duration::from_secs(60))); - let original = state(&backend, stream_id).await; - append(&backend, &basin, &stream, record()).await; - assert_eq!(state(&backend, stream_id).await, original); - assert_eq!(scheduled(&backend, stream_id).await, check); - let mut old = backend - .db - .scan(kv::key_type_range(kv::KeyType::StreamDeleteOnEmptyDeadline)) - .await - .unwrap(); - assert!(old.next().await.unwrap().is_none()); - backend.close().await.unwrap(); - } - - #[rstest::rstest] - #[case::increase(3600, true)] - #[case::decrease(10, false)] - #[tokio::test] - async fn retry_uses_record_expiration_across_retention_changes( - #[case] new_age: u64, - #[case] append_after: bool, - ) { - let backend = test_backend().await; - let initial_age = if append_after { 10 } else { 3600 }; - let (basin, stream) = create_stream( - &backend, - config(60, RetentionPolicy::Age(Duration::from_secs(initial_age))), - ) - .await; - let stream_id = StreamId::new(&basin, &stream); - let lower_bound = TimestampSecs::after(Duration::from_secs(3600)); - append(&backend, &basin, &stream, record()).await; - let original = state(&backend, stream_id).await; - configure_retention(&backend, &basin, &stream, new_age).await; - if append_after { - append(&backend, &basin, &stream, record()).await; - } - assert_eq!(state(&backend, stream_id).await, original); - let pending = due(&backend, stream_id).await; - backend.process_stream_doe(pending).await.unwrap(); - let replacement = scheduled(&backend, stream_id).await; - assert_ne!(replacement.id, pending.check.id); - assert!( - replacement.at >= lower_bound, - "must preserve a check for the actual stored expiration" - ); - backend.close().await.unwrap(); - } - - #[rstest::rstest] - #[case::never_written(false)] - #[case::expired_records(true)] - #[tokio::test] - async fn empty_old_stream_is_deleted(#[case] written: bool) { - let backend = test_backend().await; - let (basin, stream) = create_stream( - &backend, - config(1, RetentionPolicy::Age(Duration::from_secs(1))), - ) - .await; - let stream_id = StreamId::new(&basin, &stream); - if written { - append(&backend, &basin, &stream, record()).await; - } - tokio::time::sleep(Duration::from_millis(1100)).await; - backend.clone().tick_stream_doe().await.unwrap(); - let meta = backend - .db_get( - kv::stream_meta::ser_key(&basin, &stream), - kv::stream_meta::deser_value, - ) - .await - .unwrap() - .unwrap(); - assert!(meta.deleted_at.is_some()); - assert!(state(&backend, stream_id).await.is_none()); - assert!( - backend - .list_pending_stream_doe(TimestampSecs::MAX) - .await - .unwrap() - .values - .is_empty() - ); - backend.clone().tick_stream_trim().await.unwrap(); - assert!( - backend - .db_get( - kv::stream_id_mapping::ser_key(stream_id), - kv::stream_id_mapping::deser_value - ) - .await - .unwrap() - .is_none() - ); - backend.close().await.unwrap(); - } - - #[rstest::rstest] - #[case::increase(3600)] - #[case::decrease(30)] - #[case::unchanged(60)] - #[case::disable(0)] - #[case::unrepresentable_cutoff(u64::MAX)] - #[tokio::test] - async fn configuration_coalesces_or_removes_the_schedule( - #[case] age: u64, - #[values(false, true)] via_ensure: bool, - ) { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - let original = state(&backend, stream_id).await; - configure_min_age(&backend, &basin, &stream, age, via_ensure).await; - if age == 0 { - assert!(state(&backend, stream_id).await.is_none()); - assert!( - backend - .list_pending_stream_doe(TimestampSecs::MAX) - .await - .unwrap() - .values - .is_empty() - ); - configure_min_age(&backend, &basin, &stream, 60, via_ensure).await; - assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); - } else if age == 60 { - assert_eq!(state(&backend, stream_id).await, original); - } else { - assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); - backend.clone().tick_stream_doe().await.unwrap(); - assert!( - scheduled(&backend, stream_id).await.at - >= TimestampSecs::now() - .saturating_add_duration(Duration::from_secs(age.saturating_sub(1))) - ); - assert!(backend.get_stream_config(basin, stream).await.is_ok()); - } - backend.close().await.unwrap(); - } - - #[tokio::test] - async fn partial_trim_wakes_parked_stream_with_finite_records_remaining() { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - append(&backend, &basin, &stream, record()).await; - backend - .process_stream_doe(due(&backend, stream_id).await) - .await - .unwrap(); - let parked = state(&backend, stream_id).await; - assert!(matches!( - parked, - Some((kv::stream_doe_state::State::Parked, _)) - )); - assert!( - backend - .list_pending_stream_doe(TimestampSecs::MAX) - .await - .unwrap() - .values - .is_empty() - ); - configure_retention(&backend, &basin, &stream, 3600).await; - append(&backend, &basin, &stream, record()).await; - assert_eq!(state(&backend, stream_id).await, parked); - append( + let basin = BasinName::from_str("doe-basin-stale").unwrap(); + let stream = StreamName::from_str("doe-stream-stale").unwrap(); + let initial_min_age = Duration::from_secs(10); + let mut config = OptionalStreamConfig::default(); + config.delete_on_empty.min_age = Some(initial_min_age); + let stream_id = seed_stream_with_meta( &backend, &basin, &stream, - Record::Command(CommandRecord::Trim(1)), + stream_meta_with_config(config, OffsetDateTime::now_utc()), ) .await; - backend.clone().tick_stream_trim().await.unwrap(); - assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); - backend.clone().tick_stream_doe().await.unwrap(); - assert!( - scheduled(&backend, stream_id).await.at - > TimestampSecs::after(Duration::from_secs(3500)) - ); - backend.close().await.unwrap(); - } - - #[rstest::rstest] - #[case::disabled(0, false)] - #[case::disabled_before_cleanup(60, false)] - #[case::legacy_only(60, true)] - #[tokio::test] - async fn trim_initializes_missing_state_only_for_enabled_empty_streams( - #[case] min_age: u64, - #[case] legacy_only: bool, - #[values(false, true)] full_trim: bool, - ) { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(min_age, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - append(&backend, &basin, &stream, record()).await; - append( - &backend, - &basin, - &stream, - Record::Command(CommandRecord::Trim(if full_trim { u64::MAX } else { 1 })), - ) - .await; - if legacy_only { - // Model an enabled stream that has not migrated yet. - let txn = backend - .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - doe::clear(&txn, stream_id).await.unwrap(); - db_txn_commit_durable(txn).await.unwrap(); - legacy(&backend, stream_id, None).await; - } else if min_age != 0 { - configure_min_age(&backend, &basin, &stream, 0, false).await; - } - assert!(state(&backend, stream_id).await.is_none()); - - backend.clone().tick_stream_trim().await.unwrap(); - let after_trim = state(&backend, stream_id).await; - if legacy_only && full_trim { - assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); - } else { - assert!(after_trim.is_none()); - assert!( - backend - .list_pending_stream_doe(TimestampSecs::MAX) - .await - .unwrap() - .values - .is_empty() - ); - } - - // Migration preserves a full trim's wake or initializes missing state. - if legacy_only { - backend.migrate_stream_doe().await.unwrap(); - if full_trim { - assert_eq!(state(&backend, stream_id).await, after_trim); - } - assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); - } - backend.close().await.unwrap(); - } - - #[tokio::test] - async fn full_trim_restores_doe_without_any_legacy_deadline() { - let backend = test_backend().await; - let (basin, stream) = create_stream(&backend, config(1, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - append(&backend, &basin, &stream, record()).await; - // Before upgrade, an infinite-retention stream normally loses its last - // deadline when it fires while the stream is nonempty. - let txn = backend + let existing_deadline = TimestampSecs::from_secs(4_242); + backend .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - doe::clear(&txn, stream_id).await.unwrap(); - db_txn_commit_durable(txn).await.unwrap(); - backend.migrate_stream_doe().await.unwrap(); - assert!(state(&backend, stream_id).await.is_none()); - - append( - &backend, - &basin, - &stream, - Record::Command(CommandRecord::Trim(u64::MAX)), - ) - .await; - backend.clone().tick_stream_trim().await.unwrap(); - assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); - tokio::time::sleep(Duration::from_millis(1100)).await; - backend.clone().tick_stream_doe().await.unwrap(); - let meta = backend - .db_get( - kv::stream_meta::ser_key(&basin, &stream), - kv::stream_meta::deser_value, + .put( + kv::stream_doe_deadline::ser_key(existing_deadline, stream_id), + kv::stream_doe_deadline::ser_value(initial_min_age), ) .await - .unwrap() .unwrap(); - assert!(meta.deleted_at.is_some()); - assert!(state(&backend, stream_id).await.is_none()); - backend.close().await.unwrap(); - } - #[rstest::rstest] - #[tokio::test] - async fn trim_wake_conflicts_with_concurrent_disablement( - #[values(false, true)] missing_state: bool, - ) { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - if missing_state { - let txn = backend - .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - doe::clear(&txn, stream_id).await.unwrap(); - db_txn_commit_durable(txn).await.unwrap(); - } - let txn = backend - .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - doe::wake_after_trim(&txn, stream_id, false).await.unwrap(); - configure_min_age(&backend, &basin, &stream, 0, false).await; - let error = StorageError::from(txn.commit().await.unwrap_err()); - assert!(error.is_transaction_conflict()); - assert!(state(&backend, stream_id).await.is_none()); - assert!( - backend - .list_pending_stream_doe(TimestampSecs::MAX) - .await - .unwrap() - .values - .is_empty() - ); - backend.close().await.unwrap(); - } - - #[rstest::rstest] - #[case::parked(TerminalTrimOutcome::Parked)] - #[case::deferred(TerminalTrimOutcome::RetryAt(TimestampSecs::MAX))] - #[tokio::test] - async fn event_retaining_same_ticket_invalidates_inflight_result( - #[values(false, true)] trim: bool, - #[case] outcome: TerminalTrimOutcome, - ) { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - append(&backend, &basin, &stream, record()).await; - if trim { - configure_retention(&backend, &basin, &stream, 3600).await; - } - let pending = due(&backend, stream_id).await; - let snapshot = backend.observe_doe_check(pending).await.unwrap().unwrap(); - if trim { - // The worker may already have observed the infinite-retention - // record and decided to park. Subsequent appends do not invalidate - // that observation, but removing the blocker must invalidate it. - append(&backend, &basin, &stream, record()).await; - append( - &backend, - &basin, - &stream, - Record::Command(CommandRecord::Trim(1)), + backend + .reconfigure_stream( + basin, + stream, + StreamReconfiguration { + delete_on_empty: Maybe::from(Some(DeleteOnEmptyReconfiguration { + min_age: Maybe::from(Some(Duration::from_secs(600))), + })), + ..Default::default() + }, ) - .await; - backend.clone().tick_stream_trim().await.unwrap(); - } else { - configure_min_age(&backend, &basin, &stream, 120, false).await; - } - assert_eq!(scheduled(&backend, stream_id).await, pending.check); - let woken = state(&backend, stream_id).await; - assert!(woken.unwrap().1 > snapshot.revision); - backend - .finish_doe_check(pending, snapshot, outcome) .await .unwrap(); - assert_eq!(state(&backend, stream_id).await, woken); - assert_eq!(scheduled(&backend, stream_id).await, pending.check); - backend.close().await.unwrap(); - } - #[tokio::test] - async fn obsolete_check_cleanup_preserves_wake_after_snapshot() { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - let pending = due(&backend, stream_id).await; - let observed = state(&backend, stream_id).await; - configure_min_age(&backend, &basin, &stream, 120, false).await; - let woken = state(&backend, stream_id).await; - assert!(woken.unwrap().1 > observed.unwrap().1); - assert_eq!(scheduled(&backend, stream_id).await, pending.check); - backend.discard_doe_check(pending, observed).await.unwrap(); - assert_eq!(state(&backend, stream_id).await, woken); - assert_eq!(scheduled(&backend, stream_id).await, pending.check); - backend.close().await.unwrap(); - } - - #[tokio::test] - async fn legacy_deadlines_only_initialize_missing_state() { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - let txn = backend - .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - doe::clear(&txn, stream_id).await.unwrap(); - db_txn_commit_durable(txn).await.unwrap(); - let old = legacy(&backend, stream_id, None).await; - let unique = legacy(&backend, stream_id, Some(42)).await; - backend.migrate_stream_doe().await.unwrap(); - assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); - assert!(backend.db.get(&old).await.unwrap().is_none()); - assert!(backend.db.get(&unique).await.unwrap().is_none()); - append(&backend, &basin, &stream, record()).await; - backend.clone().tick_stream_doe().await.unwrap(); - let parked = state(&backend, stream_id).await; - assert!(matches!( - parked, - Some((kv::stream_doe_state::State::Parked, _)) - )); - legacy(&backend, stream_id, Some(43)).await; - backend.migrate_stream_doe().await.unwrap(); - assert_eq!(state(&backend, stream_id).await, parked); - assert!( - backend - .list_pending_stream_doe(TimestampSecs::MAX) - .await - .unwrap() - .values - .is_empty() - ); - backend.close().await.unwrap(); - } - - #[tokio::test] - async fn migration_drains_pages_without_rewriting_schedules() { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - let before = state(&backend, stream_id).await; - let mut batch = slatedb::WriteBatch::new(); - for id in 0..=PENDING_LIST_LIMIT { - batch.put( - kv::stream_doe_deadline::ser_key(TimestampSecs::MAX, stream_id, Some(id as u128)), - [], - ); - } - backend.db.write(batch).assert_durable().await; - run_tick( - "stream-delete-on-empty", - &|backend: &Backend| backend.clone().tick_stream_doe(), - &backend, - ) - .await; - let mut remaining = backend - .db - .scan(kv::key_type_range(kv::KeyType::StreamDeleteOnEmptyDeadline)) - .await - .unwrap(); - assert!(remaining.next().await.unwrap().is_none()); - assert_eq!(state(&backend, stream_id).await, before); - scheduled(&backend, stream_id).await; - backend.close().await.unwrap(); - } - - #[tokio::test] - async fn migration_batches_cleanup_across_streams() { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let scheduled_id = StreamId::new(&basin, &stream); - let before = state(&backend, scheduled_id).await; - let disabled: StreamName = "disabled".parse().unwrap(); - let deleted: StreamName = "deleted".parse().unwrap(); - for (stream, min_age) in [(&disabled, 0), (&deleted, 60)] { - backend - .provision_stream( - basin.clone(), - stream.clone(), - config(min_age, RetentionPolicy::Infinite()), - ProvisionMode::Ensure, - ) - .await - .unwrap(); - } - backend - .delete_stream(basin.clone(), deleted.clone()) - .await - .unwrap(); - let disabled_id = StreamId::new(&basin, &disabled); - let deleted_id = StreamId::new(&basin, &deleted); - let missing_id = StreamId::new(&basin, &"missing".parse().unwrap()); - let deleted_state = state(&backend, deleted_id).await; - let mut keys = Vec::new(); - for stream_id in [scheduled_id, disabled_id, deleted_id, missing_id] { - for id in 0..3 { - keys.push(legacy(&backend, stream_id, Some(id)).await); - } - } - let before_seq = backend.db.snapshot().await.unwrap().seq(); - backend.migrate_stream_doe().await.unwrap(); + let entries = list_doe_entries(&backend).await; assert_eq!( - backend.db.snapshot().await.unwrap().seq(), - before_seq + 1, - "all cleanup-only streams should share one durable batch" + entries, + vec![(existing_deadline, stream_id, initial_min_age)] ); - for key in keys { - assert!(backend.db.get(key).await.unwrap().is_none()); - } - assert_eq!(state(&backend, scheduled_id).await, before); - assert_eq!(state(&backend, deleted_id).await, deleted_state); - assert!(state(&backend, disabled_id).await.is_none()); - assert!(state(&backend, missing_id).await.is_none()); - backend.close().await.unwrap(); - } - - #[rstest::rstest] - #[tokio::test] - async fn migration_revalidates_configuration_after_snapshot(#[values(0, 120)] min_age: u64) { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - let txn = backend - .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - doe::clear(&txn, stream_id).await.unwrap(); - db_txn_commit_durable(txn).await.unwrap(); - let key = legacy(&backend, stream_id, None).await; - let snapshot = backend.db.snapshot().await.unwrap(); - configure_min_age(&backend, &basin, &stream, min_age, false).await; - let configured = state(&backend, stream_id).await; - backend - .migrate_doe_deadlines(&snapshot, stream_id, vec![key.clone()]) - .await - .unwrap(); - assert_eq!(state(&backend, stream_id).await, configured); - assert!(backend.db.get(key).await.unwrap().is_none()); - backend.close().await.unwrap(); - } - - #[tokio::test] - async fn concurrent_wake_conflicts_with_completion_transaction() { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - let pending = due(&backend, stream_id).await; - let txn = backend - .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - let previous = doe::state(&txn, stream_id).await.unwrap(); - doe::replace( - &txn, - stream_id, - previous, - Some(kv::stream_doe_state::State::Parked), - ) - .unwrap(); - // The same ticket is retained, but the scheduler revision changes. - let wake = backend - .db - .begin(IsolationLevel::SerializableSnapshot) - .await - .unwrap(); - doe::schedule(&wake, stream_id, TimestampSecs::now()) - .await - .unwrap(); - db_txn_commit_durable(wake).await.unwrap(); - let error = txn.commit().await.unwrap_err(); - assert_eq!(error.kind(), slatedb::ErrorKind::Transaction); - assert_eq!(scheduled(&backend, stream_id).await, pending.check); - backend.close().await.unwrap(); - } - - #[tokio::test] - async fn full_trim_wakes_parked_stream_and_waits_for_minimum_age() { - let backend = test_backend().await; - let (basin, stream) = - create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; - let stream_id = StreamId::new(&basin, &stream); - append(&backend, &basin, &stream, record()).await; - backend - .process_stream_doe(due(&backend, stream_id).await) - .await - .unwrap(); - assert!(matches!( - state(&backend, stream_id).await, - Some((kv::stream_doe_state::State::Parked, _)) - )); - append( - &backend, - &basin, - &stream, - Record::Command(CommandRecord::Trim(u64::MAX)), - ) - .await; - backend.clone().tick_stream_trim().await.unwrap(); - assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); - backend.clone().tick_stream_doe().await.unwrap(); - assert!(scheduled(&backend, stream_id).await.at > TimestampSecs::now()); - let meta = backend - .db_get( - kv::stream_meta::ser_key(&basin, &stream), - kv::stream_meta::deser_value, - ) - .await - .unwrap() - .unwrap(); - assert!(meta.deleted_at.is_none()); - backend.close().await.unwrap(); - } - - #[rstest::rstest] - #[case::deletion_pending(TerminalTrimOutcome::DeletionPending)] - #[case::parked(TerminalTrimOutcome::Parked)] - #[tokio::test] - async fn stale_work_cannot_delete_or_reschedule_recreated_stream( - #[case] outcome: TerminalTrimOutcome, - ) { - let backend = test_backend().await; - let configuration = config(60, RetentionPolicy::Infinite()); - let (basin, stream) = create_stream(&backend, configuration.clone()).await; - let stream_id = StreamId::new(&basin, &stream); - let pending = due(&backend, stream_id).await; - let snapshot = backend.observe_doe_check(pending).await.unwrap().unwrap(); - backend - .delete_stream(basin.clone(), stream.clone()) - .await - .unwrap(); - backend.clone().tick_stream_trim().await.unwrap(); - backend - .provision_stream( - basin.clone(), - stream.clone(), - configuration, - ProvisionMode::Ensure, - ) - .await - .unwrap(); - let recreated = state(&backend, stream_id).await; - let client = backend - .streamer_client_guarded(&basin, &stream) - .await - .unwrap(); - assert_eq!( - client - .terminal_trim(TerminalTrimCondition::DeleteOnEmpty { - expected_stream_creation_seq: snapshot.creation_seq, - expected_config_seq: snapshot.config_seq, - }) - .await - .unwrap(), - TerminalTrimOutcome::Obsolete - ); - backend - .finish_doe_check(pending, snapshot, outcome) - .await - .unwrap(); - backend.process_stream_doe(pending).await.unwrap(); - legacy(&backend, stream_id, None).await; - backend.migrate_stream_doe().await.unwrap(); - assert_eq!(state(&backend, stream_id).await, recreated); - scheduled(&backend, stream_id).await; - backend.close().await.unwrap(); } } diff --git a/lite/src/backend/bgtasks/stream_trim.rs b/lite/src/backend/bgtasks/stream_trim.rs index db377d0c..2a1f332d 100644 --- a/lite/src/backend/bgtasks/stream_trim.rs +++ b/lite/src/backend/bgtasks/stream_trim.rs @@ -3,19 +3,13 @@ use std::ops::RangeTo; use futures::{StreamExt, stream}; use s2_common::{record::NonZeroSeqNum, resources::Page}; use slatedb::{ - IsolationLevel, WriteBatch, + WriteBatch, config::{DurabilityLevel, ScanOptions}, }; use tracing::instrument; -use super::PageProgress; use crate::{ - backend::{ - Backend, doe, - error::StorageError, - kv, - store::{db_txn_commit_durable, db_txn_get, db_txn_get_with}, - }, + backend::{Backend, error::StorageError, kv, store::db_txn_get}, stream_id::StreamId, }; @@ -23,34 +17,27 @@ const PENDING_LIST_LIMIT: usize = 128; const CONCURRENCY: usize = 4; const DELETE_BATCH_SIZE: usize = 10_000; -#[derive(Debug, Clone, Copy)] -struct PendingTrim { - stream_id: StreamId, - trim_point: RangeTo, - /// Commit sequence of the observed marker; bounds the records this job may delete. - marker_seq: u64, -} - impl Backend { pub(super) async fn tick_stream_trim(self) -> Result { let page = self.list_stream_trim_pending().await?; if page.values.is_empty() { return Ok(page.has_more); } - let mut progress = PageProgress::new(page.has_more); let mut processed = stream::iter(page.values) - .map(|pending| { + .map(|(stream_id, trim_point)| { let backend = self.clone(); - async move { backend.process_trim(pending).await } + async move { backend.process_trim(stream_id, trim_point).await } }) .buffer_unordered(CONCURRENCY); while let Some(result) = processed.next().await { - progress.record_result(result, StorageError::is_transaction_conflict)?; + result?; } - Ok(progress.should_continue()) + Ok(page.has_more) } - async fn list_stream_trim_pending(&self) -> Result, StorageError> { + async fn list_stream_trim_pending( + &self, + ) -> Result)>, StorageError> { let scan_opts = ScanOptions { durability_filter: DurabilityLevel::Remote, ..Default::default() @@ -63,11 +50,7 @@ impl Backend { while let Some(kv) = it.next().await? { let stream_id = kv::stream_trim_point::deser_key(kv.key)?; let trim_point = kv::stream_trim_point::deser_value(kv.value)?; - pending.push(PendingTrim { - stream_id, - trim_point, - marker_seq: kv.seq, - }); + pending.push((stream_id, trim_point)); if pending.len() >= PENDING_LIST_LIMIT { return Ok(Page::new(pending, true)); } @@ -75,15 +58,26 @@ impl Backend { Ok(Page::new(pending, false)) } - async fn process_trim(&self, pending: PendingTrim) -> Result<(), StorageError> { - let has_remaining_records = self.delete_records(pending).await?; - self.finalize_trim(pending, has_remaining_records).await + async fn process_trim( + &self, + stream_id: StreamId, + trim_point: RangeTo, + ) -> Result<(), StorageError> { + let has_remaining_records = self.delete_records(stream_id, trim_point).await?; + if trim_point.end < NonZeroSeqNum::MAX && !has_remaining_records { + self.arm_doe_on_full_trim(stream_id).await?; + } + self.finalize_trim(stream_id, trim_point).await?; + Ok(()) } - /// Return whether records remain beyond the trim point. #[instrument(ret, err, skip(self))] - async fn delete_records(&self, pending: PendingTrim) -> Result { - let prefix = kv::stream_record_timestamp::ser_key_prefix(pending.stream_id); + async fn delete_records( + &self, + stream_id: StreamId, + trim_point: RangeTo, + ) -> Result { + let prefix = kv::stream_record_timestamp::ser_key_prefix(stream_id); let scan_opts = ScanOptions { durability_filter: DurabilityLevel::Remote, ..Default::default() @@ -93,69 +87,67 @@ impl Backend { .scan_prefix_with_options(prefix, .., &scan_opts) .await?; let mut batch = WriteBatch::new(); - let mut batch_size = 0; + let mut batch_size = 0usize; let mut has_remaining_records = false; while let Some(kv) = it.next().await? { let (deser_stream_id, pos) = kv::stream_record_timestamp::deser_key(kv.key.clone())?; - debug_assert_eq!(deser_stream_id, pending.stream_id); - // Name reuse requires all old records to be durably deleted first. - // A stale job may see records from the recreated stream; stop before - // deleting records committed after the trim marker it observed. - if pos.seq_num >= pending.trim_point.end.get() || kv.seq > pending.marker_seq { + debug_assert_eq!(deser_stream_id, stream_id); + if pos.seq_num >= trim_point.end.get() { has_remaining_records = true; break; } batch.delete(kv.key); - batch.delete(kv::stream_record_data::ser_key(pending.stream_id, pos)); + batch.delete(kv::stream_record_data::ser_key(stream_id, pos)); batch_size += 1; if batch_size >= DELETE_BATCH_SIZE { - self.db.write(batch).await?.await_durable().await?; + self.db.write(batch).await?; batch = WriteBatch::new(); batch_size = 0; } } - if !batch.is_empty() { - self.db.write(batch).await?.await_durable().await?; + if batch_size > 0 { + self.db.write(batch).await?; } Ok(has_remaining_records) } - #[instrument(skip(self))] + #[instrument(ret, err, skip(self))] async fn finalize_trim( &self, - pending: PendingTrim, - has_remaining_records: bool, + stream_id: StreamId, + trim_point: RangeTo, ) -> Result<(), StorageError> { - let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; - let trim_point_key = kv::stream_trim_point::ser_key(pending.stream_id); - let current_seq = db_txn_get_with(&txn, &trim_point_key, |entry| Ok(entry.seq)).await?; - // A tick can exit on error while its submitted cleanup is still running. - // The next durable scan can capture the old marker before cleanup becomes - // durable. Cleanup removes the old marker and metadata before name reuse, - // but this scanned work can remain queued across recreation. The recreated - // stream's marker may have the same trim point, so compare commit sequences. - if current_seq != Some(pending.marker_seq) { + let trim_point_key = kv::stream_trim_point::ser_key(stream_id); + let txn = self + .db + .begin(slatedb::IsolationLevel::SerializableSnapshot) + .await?; + let is_full_delete = trim_point == ..NonZeroSeqNum::MAX; + let Some(current_trim_point) = db_txn_get( + &txn, + trim_point_key.clone(), + kv::stream_trim_point::deser_value, + ) + .await? + else { + return Ok(()); + }; + if current_trim_point != trim_point { return Ok(()); } - let is_terminal_trim = pending.trim_point == ..NonZeroSeqNum::MAX; txn.delete(trim_point_key)?; - if is_terminal_trim { - let id_mapping_key = kv::stream_id_mapping::ser_key(pending.stream_id); + if is_full_delete { + let id_mapping_key = kv::stream_id_mapping::ser_key(stream_id); if let Some((basin, stream)) = db_txn_get(&txn, &id_mapping_key, kv::stream_id_mapping::deser_value).await? { txn.delete(kv::stream_meta::ser_key(&basin, &stream))?; txn.delete(id_mapping_key)?; } - txn.delete(kv::stream_tail_position::ser_key(pending.stream_id))?; - txn.delete(kv::stream_fencing_token::ser_key(pending.stream_id))?; - doe::clear(&txn, pending.stream_id).await?; - } else { - // A partial trim may remove the infinite-retention record that - // parked DOE while leaving finite-retention records behind. - doe::wake_after_trim(&txn, pending.stream_id, has_remaining_records).await?; + txn.delete(kv::stream_tail_position::ser_key(stream_id))?; + txn.delete(kv::stream_fencing_token::ser_key(stream_id))?; } - db_txn_commit_durable(txn).await?; + txn.commit().await?; Ok(()) } } @@ -178,10 +170,7 @@ mod tests { use time::OffsetDateTime; use super::super::tests::test_backend; - use crate::{ - backend::{kv, test_util::DbWriteTestExt as _}, - stream_id::StreamId, - }; + use crate::{backend::kv, stream_id::StreamId}; fn test_record() -> Metered { let record = Record::try_from_parts(vec![], Bytes::from_static(b"trim-test")).unwrap(); @@ -209,16 +198,16 @@ mod tests { kv::stream_record_data::ser_key(stream_id, pos), kv::stream_record_data::ser_value(metered.as_ref()), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( kv::stream_record_timestamp::ser_key(stream_id, pos), kv::stream_record_timestamp::ser_value(), ) - .assert_durable() - .await; + .await + .unwrap(); } backend @@ -227,8 +216,8 @@ mod tests { kv::stream_trim_point::ser_key(stream_id), kv::stream_trim_point::ser_value(trim_point(3)), ) - .assert_durable() - .await; + .await + .unwrap(); backend.clone().tick_stream_trim().await.unwrap(); @@ -286,16 +275,16 @@ mod tests { kv::stream_meta::ser_key(&basin, &stream), kv::stream_meta::ser_value(&meta), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( kv::stream_id_mapping::ser_key(stream_id), kv::stream_id_mapping::ser_value(&basin, &stream), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( @@ -305,8 +294,8 @@ mod tests { timestamp: 1234, }), ) - .assert_durable() - .await; + .await + .unwrap(); let token = FencingToken::from_str("token-1").unwrap(); backend .db @@ -314,8 +303,8 @@ mod tests { kv::stream_fencing_token::ser_key(stream_id), kv::stream_fencing_token::ser_value(&token), ) - .assert_durable() - .await; + .await + .unwrap(); for seq in 0..3 { let pos = StreamPosition { @@ -328,16 +317,16 @@ mod tests { kv::stream_record_data::ser_key(stream_id, pos), kv::stream_record_data::ser_value(metered.as_ref()), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( kv::stream_record_timestamp::ser_key(stream_id, pos), kv::stream_record_timestamp::ser_value(), ) - .assert_durable() - .await; + .await + .unwrap(); } backend @@ -346,8 +335,8 @@ mod tests { kv::stream_trim_point::ser_key(stream_id), kv::stream_trim_point::ser_value(trim_point(SeqNum::MAX)), ) - .assert_durable() - .await; + .await + .unwrap(); backend.clone().tick_stream_trim().await.unwrap(); @@ -402,6 +391,35 @@ mod tests { } } + #[tokio::test] + async fn stream_trim_skips_stale_trim_point() { + let backend = test_backend().await; + let stream_id: StreamId = [9u8; StreamId::LEN].into(); + + backend + .db + .put( + kv::stream_trim_point::ser_key(stream_id), + kv::stream_trim_point::ser_value(trim_point(10)), + ) + .await + .unwrap(); + + backend + .finalize_trim(stream_id, trim_point(5)) + .await + .unwrap(); + + let current = backend + .db + .get(kv::stream_trim_point::ser_key(stream_id)) + .await + .unwrap() + .expect("trim point should remain"); + let decoded = kv::stream_trim_point::deser_value(current).unwrap(); + assert_eq!(decoded, trim_point(10)); + } + #[tokio::test] async fn stream_trim_paginates_pending_list() { let backend = test_backend().await; @@ -417,7 +435,7 @@ mod tests { kv::stream_trim_point::ser_value(trim_point(1)), ); } - backend.db.write(batch).assert_durable().await; + backend.db.write(batch).await.unwrap(); let has_more = backend.clone().tick_stream_trim().await.unwrap(); assert!(has_more); @@ -454,24 +472,24 @@ mod tests { kv::stream_record_data::ser_key(stream_id, pos), kv::stream_record_data::ser_value(metered.as_ref()), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( kv::stream_record_timestamp::ser_key(stream_id, pos), kv::stream_record_timestamp::ser_value(), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( kv::stream_trim_point::ser_key(stream_id), kv::stream_trim_point::ser_value(trim_point(1)), ) - .assert_durable() - .await; + .await + .unwrap(); backend.clone().tick_stream_trim().await.unwrap(); @@ -496,6 +514,110 @@ mod tests { assert!(trim_point.is_none()); } + #[tokio::test] + async fn stream_trim_does_not_touch_other_streams() { + let backend = test_backend().await; + let stream_id_a: StreamId = [1u8; StreamId::LEN].into(); + let stream_id_b: StreamId = [2u8; StreamId::LEN].into(); + let metered = test_record(); + + for seq in 0..4 { + let pos_a = StreamPosition { + seq_num: seq, + timestamp: 1000 + seq, + }; + backend + .db + .put( + kv::stream_record_data::ser_key(stream_id_a, pos_a), + kv::stream_record_data::ser_value(metered.as_ref()), + ) + .await + .unwrap(); + backend + .db + .put( + kv::stream_record_timestamp::ser_key(stream_id_a, pos_a), + kv::stream_record_timestamp::ser_value(), + ) + .await + .unwrap(); + + let pos_b = StreamPosition { + seq_num: seq, + timestamp: 2000 + seq, + }; + backend + .db + .put( + kv::stream_record_data::ser_key(stream_id_b, pos_b), + kv::stream_record_data::ser_value(metered.as_ref()), + ) + .await + .unwrap(); + backend + .db + .put( + kv::stream_record_timestamp::ser_key(stream_id_b, pos_b), + kv::stream_record_timestamp::ser_value(), + ) + .await + .unwrap(); + } + + backend + .db + .put( + kv::stream_trim_point::ser_key(stream_id_a), + kv::stream_trim_point::ser_value(trim_point(2)), + ) + .await + .unwrap(); + + backend.clone().tick_stream_trim().await.unwrap(); + + for seq in 0..4 { + let pos_a = StreamPosition { + seq_num: seq, + timestamp: 1000 + seq, + }; + let data_a = backend + .db + .get(kv::stream_record_data::ser_key(stream_id_a, pos_a)) + .await + .unwrap(); + let timestamp_a = backend + .db + .get(kv::stream_record_timestamp::ser_key(stream_id_a, pos_a)) + .await + .unwrap(); + if seq < 2 { + assert!(data_a.is_none()); + assert!(timestamp_a.is_none()); + } else { + assert!(data_a.is_some()); + assert!(timestamp_a.is_some()); + } + + let pos_b = StreamPosition { + seq_num: seq, + timestamp: 2000 + seq, + }; + let data_b = backend + .db + .get(kv::stream_record_data::ser_key(stream_id_b, pos_b)) + .await + .unwrap(); + let timestamp_b = backend + .db + .get(kv::stream_record_timestamp::ser_key(stream_id_b, pos_b)) + .await + .unwrap(); + assert!(data_b.is_some()); + assert!(timestamp_b.is_some()); + } + } + #[tokio::test] async fn stream_trim_large_batch_flushes() { let backend = test_backend().await; @@ -523,7 +645,7 @@ mod tests { kv::stream_trim_point::ser_key(stream_id), kv::stream_trim_point::ser_value(trim_point(total)), ); - backend.db.write(batch).assert_durable().await; + backend.db.write(batch).await.unwrap(); backend.clone().tick_stream_trim().await.unwrap(); @@ -556,149 +678,47 @@ mod tests { } #[tokio::test] - async fn stale_deletion_work_preserves_recreated_stream() { - use s2_common::{ - config::OptionalStreamConfig, - resources::ProvisionMode, - stream::{AppendInput, AppendRecord, AppendRecordParts}, - }; - - use crate::backend::error::{ProvisionStreamError, ReconfigureStreamError}; + async fn finalize_trim_no_trim_point_noop() { let backend = test_backend().await; - let (basin, stream) = - crate::backend::test_util::create_stream(&backend, OptionalStreamConfig::default()) - .await; - let stream_id = StreamId::new(&basin, &stream); + let stream_id: StreamId = [4u8; StreamId::LEN].into(); + backend - .open_for_check_tail(&basin, &stream) - .await - .unwrap() - .check_tail() + .finalize_trim(stream_id, trim_point(5)) .await .unwrap(); - // Pause stream deletion after it submits terminal trim, before it marks metadata. - let mut deletion = Box::pin(backend.delete_stream(basin.clone(), stream.clone())); - assert!(futures::poll!(&mut deletion).is_pending()); - tokio::time::timeout(std::time::Duration::from_secs(5), async { - loop { - if backend - .db_get( - kv::stream_trim_point::ser_key(stream_id), - kv::stream_trim_point::deser_value, - ) - .await - .unwrap() - == Some(..NonZeroSeqNum::MAX) - { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - // A crash here leaves a durable terminal trim with unmarked metadata. - // Provisioning and reconfiguration must reject updates that the pending trim would erase. - assert!(matches!( - backend - .provision_stream( - basin.clone(), - stream.clone(), - OptionalStreamConfig::default(), - ProvisionMode::Ensure - ) - .await, - Err(ProvisionStreamError::StreamDeletionPending(_)) - )); - assert!(matches!( - backend - .reconfigure_stream(basin.clone(), stream.clone(), Default::default()) - .await, - Err(ReconfigureStreamError::StreamDeletionPending(_)) - )); - - let stale_trim = backend - .list_stream_trim_pending() + let trim_point = backend + .db + .get(kv::stream_trim_point::ser_key(stream_id)) .await - .unwrap() - .values - .pop() .unwrap(); - backend.clone().tick_stream_trim().await.unwrap(); + assert!(trim_point.is_none()); + } + + #[tokio::test] + async fn finalize_trim_clears_matching_trim_point() { + let backend = test_backend().await; + let stream_id: StreamId = [5u8; StreamId::LEN].into(); + backend - .provision_stream( - basin.clone(), - stream.clone(), - OptionalStreamConfig::default(), - ProvisionMode::CreateOnly { - request_token: None, - }, + .db + .put( + kv::stream_trim_point::ser_key(stream_id), + kv::stream_trim_point::ser_value(trim_point(5)), ) .await .unwrap(); - deletion.await.unwrap(); - let meta = backend - .db_get( - kv::stream_meta::ser_key(&basin, &stream), - kv::stream_meta::deser_value, - ) - .await - .unwrap() - .unwrap(); - assert!( - meta.deleted_at.is_none(), - "the old deletion request must not mark the recreated stream as deleted" - ); - - let record: AppendRecord = AppendRecordParts { - timestamp: None, - record: Record::try_from_parts(vec![], Bytes::from_static(b"recreated stream")) - .unwrap() - .metered(), - } - .try_into() - .unwrap(); - let ack = backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) - .await - .unwrap() - .append(AppendInput { - records: vec![record].try_into().unwrap(), - match_seq_num: None, - fencing_token: None, - }) + backend + .finalize_trim(stream_id, trim_point(5)) .await .unwrap(); - // A previous tick can have scanned the old marker before cleanup became durable. - backend.process_trim(stale_trim).await.unwrap(); - assert!( - backend - .db_get( - kv::stream_record_data::ser_key(stream_id, ack.start), - kv::stream_record_data::deser_value, - ) - .await - .unwrap() - .is_some(), - "old trim work must not delete records from the recreated stream" - ); - // Even another terminal marker belongs to different work after recreation. - backend.delete_stream(basin, stream).await.unwrap(); - backend.process_trim(stale_trim).await.unwrap(); - assert_eq!( - backend - .db_get( - kv::stream_trim_point::ser_key(stream_id), - kv::stream_trim_point::deser_value, - ) - .await - .unwrap(), - Some(..NonZeroSeqNum::MAX), - "old trim work must not finalize the recreated stream's deletion" - ); - backend.close().await.unwrap(); + let trim_point = backend + .db + .get(kv::stream_trim_point::ser_key(stream_id)) + .await + .unwrap(); + assert!(trim_point.is_none()); } } diff --git a/lite/src/backend/core.rs b/lite/src/backend/core.rs index 25359c60..04eace85 100644 --- a/lite/src/backend/core.rs +++ b/lite/src/backend/core.rs @@ -8,12 +8,13 @@ use futures::{ }; use s2_common::{ basin::BasinName, - config::{BasinConfig, OptionalStreamConfig, StreamConfig}, + config::{BasinConfig, OptionalStreamConfig}, encryption::{EncryptionAlgorithm, EncryptionSpec}, record::{NonZeroSeqNum, SeqNum, StreamPosition}, resources::ProvisionMode, stream::StreamName, }; +use slatedb::config::{DurabilityLevel, ReadOptions, ScanOptions}; use tokio::sync::{Semaphore, broadcast}; use super::{ @@ -25,9 +26,7 @@ use super::{ StreamerMissingInActionError, TransactionConflictError, }, kv, - store::db_snapshot_get_with, streamer::{GuardedStreamerClient, StreamerClient, StreamerGenerationId}, - timestamp::TimestampSecs, }; use crate::{backend::bgtasks::BgtaskTrigger, stream_id::StreamId}; @@ -38,7 +37,6 @@ enum StreamerClientSlot { Initializing { generation_id: StreamerGenerationId, future: StreamerInitFuture, - pending_config: Option<(u64, StreamConfig)>, }, Ready { client: StreamerClient, @@ -73,14 +71,6 @@ impl Backend { } } - /// Flush memtables to L0 and close the database. - /// - /// Call after draining HTTP requests to reduce WAL replay on restart. - /// Dropping the backend does not close SlateDB. - pub async fn close(&self) -> Result<(), slatedb::Error> { - self.db.close().await - } - pub(super) fn bgtask_trigger(&self, trigger: BgtaskTrigger) { let _ = self.bgtask_trigger_tx.send(trigger); } @@ -114,74 +104,46 @@ impl Backend { basin: BasinName, stream: StreamName, ) -> Result { - // Read one consistent, durable view of the stream. - let snapshot = self.db.snapshot().await.map_err(StorageError::from)?; - self.await_durable_seq(snapshot.seq()).await?; let stream_id = StreamId::new(&basin, &stream); - let (meta, persisted_tail, fencing_token, trim_point, stream_creation_seq) = tokio::try_join!( - db_snapshot_get_with( - &snapshot, + let (meta, persisted_tail, fencing_token, trim_point) = tokio::try_join!( + self.db_get( kv::stream_meta::ser_key(&basin, &stream), - |entry| { Ok((kv::stream_meta::deser_value(entry.value)?, entry.seq)) } + kv::stream_meta::deser_value, ), - db_snapshot_get_with( - &snapshot, - kv::stream_tail_position::ser_key(stream_id), - |entry| { - Ok(( - kv::stream_tail_position::deser_value(entry.value)?, - TimestampSecs::from_millis(entry.create_ts), - )) - } - ), - db_snapshot_get_with( - &snapshot, + self.load_persisted_stream_tail(stream_id), + self.db_get( kv::stream_fencing_token::ser_key(stream_id), - |entry| kv::stream_fencing_token::deser_value(entry.value), + kv::stream_fencing_token::deser_value, ), - db_snapshot_get_with( - &snapshot, + self.db_get( kv::stream_trim_point::ser_key(stream_id), - |entry| kv::stream_trim_point::deser_value(entry.value), - ), - db_snapshot_get_with( - &snapshot, - kv::stream_id_mapping::ser_key(stream_id), - |entry| Ok(entry.seq) - ), + kv::stream_trim_point::deser_value, + ) )?; - let Some((meta, config_seq)) = meta else { + let Some(meta) = meta else { return Err(StreamNotFoundError { basin, stream }.into()); }; let (tail_pos, last_tail_write_timestamp) = - persisted_tail.unwrap_or((StreamPosition::MIN, TimestampSecs::ZERO)); + persisted_tail.unwrap_or((StreamPosition::MIN, kv::timestamp::TimestampSecs::ZERO)); - if meta.deleted_at.is_some() || trim_point == Some(..NonZeroSeqNum::MAX) { - return Err(StreamDeletionPendingError.into()); - } - - self.assert_no_records_following_tail(&snapshot, stream_id, &basin, &stream, tail_pos) + self.assert_no_records_following_tail(stream_id, &basin, &stream, tail_pos) .await?; let fencing_token = fencing_token.unwrap_or_default(); - let stream_creation_seq = stream_creation_seq.ok_or_else(|| { - StorageError::InvariantViolation(format!( - "live stream `{basin}/{stream}` has no ID mapping" - )) - })?; + if trim_point == Some(..NonZeroSeqNum::MAX) { + return Err(StreamDeletionPendingError.into()); + } let streamer_slots = self.streamer_slots.clone(); Ok(super::streamer::Spawner { generation_id, db: self.db.clone(), stream_id, - stream_creation_seq, config: meta.config, - config_seq, cipher: meta.cipher, tail_pos, last_tail_write_timestamp, @@ -198,9 +160,29 @@ impl Backend { })) } + async fn load_persisted_stream_tail( + &self, + stream_id: StreamId, + ) -> Result, StorageError> { + let read_opts = ReadOptions { + durability_filter: DurabilityLevel::Remote, + ..Default::default() + }; + let Some(entry) = self + .db + .get_key_value_with_options(kv::stream_tail_position::ser_key(stream_id), &read_opts) + .await? + else { + return Ok(None); + }; + Ok(Some(( + kv::stream_tail_position::deser_value(entry.value)?, + kv::timestamp::TimestampSecs::from_millis(entry.create_ts), + ))) + } + async fn assert_no_records_following_tail( &self, - snapshot: &slatedb::DbSnapshot, stream_id: StreamId, basin: &BasinName, stream: &StreamName, @@ -211,7 +193,14 @@ impl Backend { seq_num: tail_pos.seq_num, timestamp: 0, }); - let mut it = snapshot.scan_prefix(prefix, start_suffix..).await?; + let scan_opts = ScanOptions { + durability_filter: DurabilityLevel::Remote, + ..Default::default() + }; + let mut it = self + .db + .scan_prefix_with_options(prefix, start_suffix.., &scan_opts) + .await?; let Some(kv) = it.next().await? else { return Ok(()); }; @@ -244,22 +233,13 @@ impl Backend { fn new_initializing_slot(self, basin: &BasinName, stream: &StreamName) -> StreamerClientSlot { let basin = basin.clone(); let stream = stream.clone(); - let stream_id = StreamId::new(&basin, &stream); let generation_id = StreamerGenerationId::next(); - // Drive initialization independently so cancelled callers cannot leave - // its snapshot pinned in a cached, unpolled future. - let future = tokio::spawn(async move { - let result = self.start_streamer(generation_id, basin, stream).await; - self.streamer_finish_initialization(stream_id, generation_id, &result); - result - }) - .map(|result| result.expect("streamer initialization task panicked")) - .boxed() - .shared(); + let future = async move { self.start_streamer(generation_id, basin, stream).await } + .boxed() + .shared(); StreamerClientSlot::Initializing { generation_id, future, - pending_config: None, } } @@ -284,13 +264,6 @@ impl Backend { if client.is_dead() { oe.remove(); } else { - if let StreamerClientSlot::Initializing { - pending_config: Some((seq, config)), - .. - } = oe.get() - { - client.advise_reconfig(*seq, config.clone()); - } oe.insert(StreamerClientSlot::Ready { client: client.clone(), }); @@ -309,35 +282,30 @@ impl Backend { basin: &BasinName, stream: &StreamName, ) -> Result { + let stream_id = StreamId::new(basin, stream); match self.streamer_client_slot(basin, stream) { - StreamerClientSlot::Initializing { future, .. } => future.await, + StreamerClientSlot::Initializing { + generation_id, + future, + } => { + let result = future.await; + self.streamer_finish_initialization(stream_id, generation_id, &result); + result + } StreamerClientSlot::Ready { client } => Ok(client), } } - pub(super) fn advise_stream_config( + pub(super) fn streamer_client_if_active( &self, basin: &BasinName, stream: &StreamName, - seq: u64, - config: StreamConfig, - ) { + ) -> Option { let stream_id = StreamId::new(basin, stream); - let Some(mut slot) = self.streamer_slots.get_mut(&stream_id) else { - return; - }; - match slot.value_mut() { - StreamerClientSlot::Ready { client } => { - client.advise_reconfig(seq, config); - } - StreamerClientSlot::Initializing { pending_config, .. } => { - if pending_config - .as_ref() - .is_none_or(|(pending_seq, _)| seq > *pending_seq) - { - *pending_config = Some((seq, config)); - } - } + let slot = self.streamer_slots.get(&stream_id)?; + match slot.value() { + StreamerClientSlot::Ready { client } if !client.is_dead() => Some(client.clone()), + _ => None, } } @@ -355,16 +323,11 @@ impl Backend { } } - /// Resolve a handle for `stream`, creating it on demand if the basin config allows. - /// - /// `stream_config` is applied over the basin's default stream configuration only if the - /// stream is being created. It must already be validated. pub(super) async fn stream_handle_with_auto_create( &self, basin: &BasinName, stream: &StreamName, - auto_create_on: AutoCreateOn, - stream_config: OptionalStreamConfig, + should_auto_create: impl FnOnce(&BasinConfig) -> bool, resolve_encryption: impl FnOnce(Option) -> Result, ) -> Result where @@ -376,66 +339,54 @@ impl Backend { + From + From, { - let client = match self.streamer_client_guarded(basin, stream).await { - Ok(client) => client, + match self.streamer_client_guarded(basin, stream).await { + Ok(client) => Ok(StreamHandle { + db: self.db.clone(), + encryption: resolve_encryption(client.cipher())?, + client, + }), Err(StreamerError::StreamNotFound(e)) => { let config = match self.get_basin_config(basin.clone()).await { Ok(config) => config, Err(GetBasinConfigError::Storage(e)) => Err(e)?, Err(GetBasinConfigError::BasinNotFound(e)) => Err(e)?, }; - if !auto_create_on.is_enabled(&config) { - return Err(e.into()); - } - if let Err(e) = self - .provision_stream( - basin.clone(), - stream.clone(), - stream_config, - ProvisionMode::CreateOnly { - request_token: None, - }, - ) - .await - { - match e { - ProvisionStreamError::Storage(e) => Err(e)?, - ProvisionStreamError::TransactionConflict(e) => Err(e)?, - ProvisionStreamError::BasinDeletionPending(e) => Err(e)?, - ProvisionStreamError::StreamDeletionPending(e) => Err(e)?, - ProvisionStreamError::BasinNotFound(e) => Err(e)?, - ProvisionStreamError::StreamAlreadyExists(_) => {} - ProvisionStreamError::Validation(e) => { - unreachable!("auto-create config is validated at the API boundary: {e}") + if should_auto_create(&config) { + if let Err(e) = self + .provision_stream( + basin.clone(), + stream.clone(), + OptionalStreamConfig::default(), + ProvisionMode::CreateOnly { + request_token: None, + }, + ) + .await + { + match e { + ProvisionStreamError::Storage(e) => Err(e)?, + ProvisionStreamError::TransactionConflict(e) => Err(e)?, + ProvisionStreamError::BasinDeletionPending(e) => Err(e)?, + ProvisionStreamError::StreamDeletionPending(e) => Err(e)?, + ProvisionStreamError::BasinNotFound(e) => Err(e)?, + ProvisionStreamError::StreamAlreadyExists(_) => {} + ProvisionStreamError::Validation(_) => { + unreachable!("auto-create uses default config") + } } } + let client = self.streamer_client_guarded(basin, stream).await?; + let encryption = resolve_encryption(client.cipher())?; + Ok(StreamHandle { + db: self.db.clone(), + encryption, + client, + }) + } else { + Err(e.into()) } - self.streamer_client_guarded(basin, stream).await? } - Err(e) => return Err(e.into()), - }; - Ok(StreamHandle { - db: self.db.clone(), - encryption: resolve_encryption(client.cipher())?, - client, - }) - } -} - -/// Which basin setting governs creating a missing stream on demand. -#[derive(Debug, Clone, Copy)] -pub(super) enum AutoCreateOn { - /// `create_stream_on_append` - Append, - /// `create_stream_on_read` - Read, -} - -impl AutoCreateOn { - fn is_enabled(self, config: &BasinConfig) -> bool { - match self { - Self::Append => config.create_stream_on_append, - Self::Read => config.create_stream_on_read, + Err(e) => Err(e.into()), } } } @@ -446,17 +397,15 @@ mod tests { use bytes::Bytes; use s2_common::{ - config::{BasinConfig, OptionalStreamConfig, StreamConfig, TimestampingMode}, + config::{BasinConfig, OptionalStreamConfig, StreamConfig}, record::{Metered, MeteredExt as _, Record, StreamPosition}, resources::ProvisionMode, - stream::{AppendInput, AppendRecord, AppendRecordParts}, }; use s2_storage::record::StoredRecord; use slatedb::{WriteBatch, object_store}; use time::OffsetDateTime; use super::*; - use crate::backend::{error::AppendError, test_util::DbWriteTestExt as _}; async fn new_test_backend() -> Backend { let object_store: Arc = @@ -468,22 +417,6 @@ mod tests { Backend::new(db, ByteSize::b(1)) } - fn append_input(body: &str) -> AppendInput { - let record = - Record::try_from_parts(vec![], Bytes::copy_from_slice(body.as_bytes())).unwrap(); - let record: AppendRecord = AppendRecordParts { - timestamp: None, - record: Metered::from(record), - } - .try_into() - .unwrap(); - AppendInput { - records: vec![record].try_into().unwrap(), - match_seq_num: None, - fencing_token: None, - } - } - #[tokio::test] #[should_panic(expected = "invariant violation: stream `testbasin1/stream1` tail_pos")] async fn start_streamer_fails_if_records_exist_after_tail_pos() { @@ -526,7 +459,7 @@ mod tests { kv::stream_record_data::ser_key(stream_id, record_pos), kv::stream_record_data::ser_value(metered_record.as_ref()), ); - backend.db.write(wb).assert_durable().await; + backend.db.write(wb).await.unwrap(); backend .start_streamer(StreamerGenerationId::next(), basin.clone(), stream.clone()) @@ -535,77 +468,45 @@ mod tests { } #[tokio::test] - async fn streamer_initialization_is_shared_and_survives_cancellation() { - let db = slatedb::Db::builder("test", Arc::new(object_store::memory::InMemory::new())) - .with_settings(slatedb::config::Settings { - flush_interval: None, - ..Default::default() - }) - .build() - .await - .unwrap(); - let backend = Backend::new(db, ByteSize::b(1)); - backend.db.put(b"unflushed", b"value").await.unwrap(); + async fn streamer_client_slot_uses_single_initializer() { + let backend = new_test_backend().await; let basin = BasinName::from_str("testbasin2").unwrap(); let stream = StreamName::from_str("stream2").unwrap(); let slot_1 = backend.streamer_client_slot(&basin, &stream); let slot_2 = backend.streamer_client_slot(&basin, &stream); - let (generation_id_1, generation_id_2, mut future_1, future_2) = match (slot_1, slot_2) { + let (generation_id_1, generation_id_2) = match (slot_1, slot_2) { ( StreamerClientSlot::Initializing { generation_id: generation_id_1, - future: future_1, .. }, StreamerClientSlot::Initializing { generation_id: generation_id_2, - future: future_2, .. }, - ) => (generation_id_1, generation_id_2, future_1, future_2), + ) => (generation_id_1, generation_id_2), _ => panic!("expected both slots to be Initializing"), }; assert_eq!(generation_id_1, generation_id_2); assert_eq!(backend.streamer_slots.len(), 1); - - // Cancel every caller while initialization waits for its snapshot to be durable. - assert!(futures::poll!(&mut future_1).is_pending()); - tokio::task::yield_now().await; - drop((future_1, future_2)); - backend.db.flush().await.unwrap(); - - // The missing stream must still finish initialization and release its snapshot. - tokio::time::timeout(std::time::Duration::from_secs(1), async { - while !backend.streamer_slots.is_empty() { - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - backend.close().await.unwrap(); } #[tokio::test] - async fn config_notification_does_not_start_streamer() { + async fn streamer_client_if_active_is_peek_only() { let backend = new_test_backend().await; let basin = BasinName::from_str("testbasin3").unwrap(); let stream = StreamName::from_str("stream3").unwrap(); - backend.advise_stream_config(&basin, &stream, 1, StreamConfig::default()); - assert!(backend.streamer_slots.is_empty()); - backend.close().await.unwrap(); - } - - #[tokio::test] - async fn initializing_streamer_receives_latest_config() { - let backend = new_test_backend().await; - let basin = BasinName::from_str("config-init").unwrap(); - let stream = StreamName::from_str("stream").unwrap(); - let stream_id = StreamId::new(&basin, &stream); backend - .provision_basin(basin.clone(), BasinConfig::default(), ProvisionMode::Ensure) + .provision_basin( + basin.clone(), + BasinConfig::default(), + ProvisionMode::CreateOnly { + request_token: None, + }, + ) .await .unwrap(); backend @@ -613,40 +514,16 @@ mod tests { basin.clone(), stream.clone(), OptionalStreamConfig::default(), - ProvisionMode::Ensure, + ProvisionMode::CreateOnly { + request_token: None, + }, ) .await .unwrap(); - // Pause initialization after reading metadata, before publishing the client. - let generation_id = StreamerGenerationId::next(); - let client = backend - .start_streamer(generation_id, basin.clone(), stream.clone()) - .await - .unwrap(); - backend.streamer_slots.insert( - stream_id, - StreamerClientSlot::Initializing { - generation_id, - future: futures::future::pending().boxed().shared(), - pending_config: None, - }, - ); - let mut config = StreamConfig::default(); - config.timestamping.mode = TimestampingMode::ClientRequire; - backend.advise_stream_config(&basin, &stream, 20, config); - backend.advise_stream_config(&basin, &stream, 10, StreamConfig::default()); - backend.streamer_finish_initialization(stream_id, generation_id, &Ok(client)); - - let handle = backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) - .await - .unwrap(); - assert!(matches!( - handle.append(append_input("missing timestamp")).await, - Err(AppendError::TimestampMissing(_)) - )); - backend.close().await.unwrap(); + assert!(backend.streamer_slots.is_empty()); + assert!(backend.streamer_client_if_active(&basin, &stream).is_none()); + assert!(backend.streamer_slots.is_empty()); } #[tokio::test] @@ -683,7 +560,6 @@ mod tests { StreamerClientSlot::Initializing { generation_id: current_generation_id, future: future.clone(), - pending_config: None, }, ); @@ -703,6 +579,26 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn concurrent_appends_auto_create_stream_without_spurious_not_found() { + use s2_common::stream::{AppendInput, AppendRecord, AppendRecordParts}; + + use crate::backend::error::AppendError; + + fn append_input(body: &str) -> AppendInput { + let record = + Record::try_from_parts(vec![], Bytes::copy_from_slice(body.as_bytes())).unwrap(); + let record: AppendRecord = AppendRecordParts { + timestamp: None, + record: Metered::from(record), + } + .try_into() + .unwrap(); + AppendInput { + records: vec![record].try_into().unwrap(), + match_seq_num: None, + fencing_token: None, + } + } + let backend = new_test_backend().await; let basin = BasinName::from_str("autocreate").unwrap(); backend @@ -727,9 +623,7 @@ mod tests { let basin = basin.clone(); let stream = stream.clone(); tokio::spawn(async move { - let handle = backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) - .await?; + let handle = backend.open_for_append(&basin, &stream, None).await?; handle.append(append_input(&format!("r{i}"))).await }) }) diff --git a/lite/src/backend/doe.rs b/lite/src/backend/doe.rs deleted file mode 100644 index 37d7ba7a..00000000 --- a/lite/src/backend/doe.rs +++ /dev/null @@ -1,124 +0,0 @@ -//! Transactional scheduling shared by configuration changes, trims, and migration. -//! Appends only postpone eligibility and do not need to update the schedule. - -use std::time::Duration; - -use slatedb::DbTransaction; - -use super::{error::StorageError, kv, store::db_txn_get, timestamp::TimestampSecs}; -use crate::stream_id::StreamId; - -/// Bound polling on active streams, while allowing known expirations to defer -/// checks much farther into the future. -pub(super) const RETRY_INTERVAL: Duration = Duration::from_secs(600); - -pub(super) async fn state( - txn: &DbTransaction, - stream_id: StreamId, -) -> Result, StorageError> { - db_txn_get( - txn, - kv::stream_doe_state::ser_key(stream_id), - kv::stream_doe_state::deser_value, - ) - .await -} - -/// Request a check without adding a second ticket. Even when keeping an earlier -/// ticket, rewrite the state to advance its commit sequence: an in-flight worker -/// must not park or postpone the stream after a concurrent trim/configuration wake. -pub(super) async fn schedule( - txn: &DbTransaction, - stream_id: StreamId, - at: TimestampSecs, -) -> Result<(), StorageError> { - let previous = state(txn, stream_id).await?; - schedule_observed(txn, stream_id, previous, at)?; - Ok(()) -} - -/// Most trims need only the state row. A full trim also restores scheduling for -/// older enabled streams whose last legacy deadline was already consumed. -pub(super) async fn wake_after_trim( - txn: &DbTransaction, - stream_id: StreamId, - has_remaining_records: bool, -) -> Result<(), StorageError> { - let previous = state(txn, stream_id).await?; - if previous.is_none() { - if has_remaining_records { - return Ok(()); - } - let Some((basin, stream)) = db_txn_get( - txn, - kv::stream_id_mapping::ser_key(stream_id), - kv::stream_id_mapping::deser_value, - ) - .await? - else { - return Ok(()); - }; - let Some(meta) = db_txn_get( - txn, - kv::stream_meta::ser_key(&basin, &stream), - kv::stream_meta::deser_value, - ) - .await? - else { - return Ok(()); - }; - if meta.deleted_at.is_some() || meta.config.delete_on_empty.min_age().is_none() { - return Ok(()); - } - } - schedule_observed(txn, stream_id, previous, TimestampSecs::now())?; - Ok(()) -} - -fn schedule_observed( - txn: &DbTransaction, - stream_id: StreamId, - previous: Option, - at: TimestampSecs, -) -> Result<(), slatedb::Error> { - let next = match previous { - Some(kv::stream_doe_state::State::Scheduled(check)) if check.at <= at => { - kv::stream_doe_state::State::Scheduled(check) - } - _ => kv::stream_doe_state::State::Scheduled(kv::stream_doe_state::Check { - at, - id: rand::random(), - }), - }; - replace(txn, stream_id, previous, Some(next)) -} - -pub(super) async fn clear(txn: &DbTransaction, stream_id: StreamId) -> Result<(), StorageError> { - let previous = state(txn, stream_id).await?; - replace(txn, stream_id, previous, None)?; - Ok(()) -} - -/// The caller must have read the state in this serializable transaction. -pub(super) fn replace( - txn: &DbTransaction, - stream_id: StreamId, - previous: Option, - next: Option, -) -> Result<(), slatedb::Error> { - if previous != next { - if let Some(kv::stream_doe_state::State::Scheduled(check)) = previous { - txn.delete(kv::stream_doe_check::ser_key(stream_id, check))?; - } - if let Some(kv::stream_doe_state::State::Scheduled(check)) = next { - txn.put(kv::stream_doe_check::ser_key(stream_id, check), [])?; - } - } - let key = kv::stream_doe_state::ser_key(stream_id); - match next { - Some(next) => txn.put(key, kv::stream_doe_state::ser_value(next))?, - None if previous.is_some() => txn.delete(key)?, - None => (), - } - Ok(()) -} diff --git a/lite/src/backend/error.rs b/lite/src/backend/error.rs index 62970c58..3c441a65 100644 --- a/lite/src/backend/error.rs +++ b/lite/src/backend/error.rs @@ -12,20 +12,12 @@ use crate::backend::kv; #[derive(Debug, Clone, thiserror::Error)] pub enum StorageError { - #[error("invariant violation: {0}")] - InvariantViolation(String), #[error("deserialization: {0}")] Deserialization(#[from] kv::DeserializationError), #[error("database: {0}")] Database(Arc), } -impl StorageError { - pub(super) fn is_transaction_conflict(&self) -> bool { - matches!(self, Self::Database(err) if err.kind() == slatedb::ErrorKind::Transaction) - } -} - impl From for StorageError { fn from(error: slatedb::Error) -> Self { StorageError::Database(Arc::new(error)) @@ -114,10 +106,8 @@ pub(super) enum AppendErrorInternal { StreamerMissingInActionError(#[from] StreamerMissingInActionError), #[error(transparent)] RequestDroppedError(#[from] RequestDroppedError), - #[error("stream deletion pending")] - StreamDeletionPending { - durability_dependency: RangeTo, - }, + #[error(transparent)] + StreamDeletionPending(#[from] StreamDeletionPendingError), #[error(transparent)] ConditionFailed(#[from] AppendConditionFailedError), #[error(transparent)] @@ -129,9 +119,6 @@ pub(super) enum AppendErrorInternal { impl AppendErrorInternal { pub fn durability_dependency(&self) -> RangeTo { match self { - Self::StreamDeletionPending { - durability_dependency, - } => *durability_dependency, Self::ConditionFailed(e) => e.durability_dependency(), Self::MaxSeqNum(e) => e.durability_dependency(), _ => ..0, @@ -203,9 +190,7 @@ impl From for AppendError { AppendError::StreamerMissingInActionError(e) } AppendErrorInternal::RequestDroppedError(e) => AppendError::RequestDroppedError(e), - AppendErrorInternal::StreamDeletionPending { .. } => { - AppendError::StreamDeletionPending(StreamDeletionPendingError) - } + AppendErrorInternal::StreamDeletionPending(e) => AppendError::StreamDeletionPending(e), AppendErrorInternal::ConditionFailed(e) => AppendError::ConditionFailed(e), AppendErrorInternal::TimestampMissing(e) => AppendError::TimestampMissing(e), AppendErrorInternal::MaxSeqNum(e) => AppendError::MaxSeqNum(e), @@ -323,7 +308,7 @@ impl From for ListStreamsError { #[derive(Debug, Clone, thiserror::Error)] pub enum ProvisionStreamError { #[error(transparent)] - Storage(StorageError), + Storage(#[from] StorageError), #[error(transparent)] TransactionConflict(#[from] TransactionConflictError), #[error(transparent)] @@ -340,16 +325,10 @@ pub enum ProvisionStreamError { impl From for ProvisionStreamError { fn from(err: slatedb::Error) -> Self { - Self::from(StorageError::from(err)) - } -} - -impl From for ProvisionStreamError { - fn from(err: StorageError) -> Self { - if err.is_transaction_conflict() { + if err.kind() == slatedb::ErrorKind::Transaction { Self::TransactionConflict(TransactionConflictError) } else { - Self::Storage(err) + Self::Storage(err.into()) } } } @@ -413,18 +392,6 @@ pub enum StreamDeleteOnEmptyError { DeleteStream(#[from] DeleteStreamError), } -impl StreamDeleteOnEmptyError { - pub(super) fn is_transaction_conflict(&self) -> bool { - match self { - Self::Storage(err) | Self::DeleteStream(DeleteStreamError::Storage(err)) => { - err.is_transaction_conflict() - } - Self::DeleteStream(DeleteStreamError::TransactionConflict(_)) => true, - _ => false, - } - } -} - #[derive(Debug, Clone, thiserror::Error)] pub enum ListBasinsError { #[error(transparent)] @@ -498,7 +465,7 @@ impl From for ReconfigureBasinError { #[derive(Debug, Clone, thiserror::Error)] pub enum ReconfigureStreamError { #[error(transparent)] - Storage(StorageError), + Storage(#[from] StorageError), #[error(transparent)] TransactionConflict(#[from] TransactionConflictError), #[error(transparent)] @@ -515,16 +482,10 @@ pub enum ReconfigureStreamError { impl From for ReconfigureStreamError { fn from(err: slatedb::Error) -> Self { - Self::from(StorageError::from(err)) - } -} - -impl From for ReconfigureStreamError { - fn from(err: StorageError) -> Self { - if err.is_transaction_conflict() { + if err.kind() == slatedb::ErrorKind::Transaction { Self::TransactionConflict(TransactionConflictError) } else { - Self::Storage(err) + Self::Storage(err.into()) } } } diff --git a/lite/src/backend/kv/mod.rs b/lite/src/backend/kv/mod.rs index 95f0e505..893abe27 100644 --- a/lite/src/backend/kv/mod.rs +++ b/lite/src/backend/kv/mod.rs @@ -1,8 +1,6 @@ pub mod basin_deletion_pending; pub mod basin_meta; -pub mod stream_doe_check; pub mod stream_doe_deadline; -pub mod stream_doe_state; pub mod stream_fencing_token; pub mod stream_id_mapping; pub mod stream_meta; @@ -10,6 +8,7 @@ pub mod stream_record_data; pub mod stream_record_timestamp; pub mod stream_tail_position; pub mod stream_trim_point; +pub mod timestamp; use std::{ops::Range, str::FromStr}; @@ -20,7 +19,7 @@ use s2_common::{ use strum::FromRepr; use thiserror::Error; -use crate::{backend::timestamp::TimestampSecs, stream_id::StreamId}; +use crate::stream_id::StreamId; #[derive(Debug, Clone, Error)] pub enum DeserializationError { @@ -50,8 +49,6 @@ pub enum KeyType { StreamRecordData = 6, StreamRecordTimestamp = 7, StreamDeleteOnEmptyDeadline = 10, - StreamDeleteOnEmptyState = 11, - StreamDeleteOnEmptyCheck = 12, } #[derive(Debug, Clone)] @@ -92,21 +89,10 @@ pub enum Key { /// Key: StreamID Timestamp SeqNum /// Value: empty StreamRecordTimestamp(StreamId, StreamPosition), - /// (SDOED) legacy schedule, consumed only to initialize the new DOE state - /// Key: TimestampSecs StreamID ScheduleID (u128, absent in legacy keys) + /// (SDOED) per-deadline-per-stream, deletable, present while pending + /// Key: TimestampSecs StreamID /// Value: MinAge seconds (u64) - StreamDeleteOnEmptyDeadline(TimestampSecs, StreamId, Option), - /// (SDOES) per-stream, updatable, optional - /// Key: StreamID - /// Value: Tag (u8: 0 = parked, 1 = scheduled). - /// Scheduled values append TimestampSecs (u32) CheckID (u128). - /// Uses the entry's SlateDB sequence as its revision. - /// State older than the current stream-ID mapping is stale. - StreamDeleteOnEmptyState(StreamId), - /// (SDOEC) per-check, immutable, deletable, time-ordered index of scheduled states - /// Key: TimestampSecs (u32) StreamID CheckID (u128) - /// Value: empty - StreamDeleteOnEmptyCheck(StreamId, stream_doe_state::Check), + StreamDeleteOnEmptyDeadline(timestamp::TimestampSecs, StreamId), } impl From for Bytes { @@ -123,12 +109,8 @@ impl From for Bytes { Key::StreamRecordTimestamp(stream_id, pos) => { stream_record_timestamp::ser_key(stream_id, pos) } - Key::StreamDeleteOnEmptyDeadline(deadline, stream_id, schedule_id) => { - stream_doe_deadline::ser_key(deadline, stream_id, schedule_id) - } - Key::StreamDeleteOnEmptyState(stream_id) => stream_doe_state::ser_key(stream_id), - Key::StreamDeleteOnEmptyCheck(stream_id, check) => { - stream_doe_check::ser_key(stream_id, check) + Key::StreamDeleteOnEmptyDeadline(deadline, stream_id) => { + stream_doe_deadline::ser_key(deadline, stream_id) } } } @@ -165,16 +147,8 @@ impl TryFrom for Key { .map(|(stream_id, pos)| Key::StreamRecordData(stream_id, pos)), KeyType::StreamRecordTimestamp => stream_record_timestamp::deser_key(bytes) .map(|(stream_id, pos)| Key::StreamRecordTimestamp(stream_id, pos)), - KeyType::StreamDeleteOnEmptyDeadline => { - stream_doe_deadline::deser_key(bytes).map(|(deadline, stream_id, schedule_id)| { - Key::StreamDeleteOnEmptyDeadline(deadline, stream_id, schedule_id) - }) - } - KeyType::StreamDeleteOnEmptyState => { - stream_doe_state::deser_key(bytes).map(Key::StreamDeleteOnEmptyState) - } - KeyType::StreamDeleteOnEmptyCheck => stream_doe_check::deser_key(bytes) - .map(|(stream_id, check)| Key::StreamDeleteOnEmptyCheck(stream_id, check)), + KeyType::StreamDeleteOnEmptyDeadline => stream_doe_deadline::deser_key(bytes) + .map(|(deadline, stream_id)| Key::StreamDeleteOnEmptyDeadline(deadline, stream_id)), } } } diff --git a/lite/src/backend/kv/stream_doe_check.rs b/lite/src/backend/kv/stream_doe_check.rs deleted file mode 100644 index 2ec02c2c..00000000 --- a/lite/src/backend/kv/stream_doe_check.rs +++ /dev/null @@ -1,78 +0,0 @@ -use std::ops::Range; - -use bytes::{Buf, BufMut, Bytes, BytesMut}; - -use super::{DeserializationError, KeyType, check_exact_size, stream_doe_state::Check}; -use crate::{backend::timestamp::TimestampSecs, stream_id::StreamId}; - -const KEY_LEN: usize = 1 + 4 + StreamId::LEN + 16; - -pub fn ser_key(stream_id: StreamId, check: Check) -> Bytes { - let mut buf = BytesMut::with_capacity(KEY_LEN); - buf.put_u8(KeyType::StreamDeleteOnEmptyCheck as u8); - buf.put_u32(check.at.as_u32()); - buf.put_slice(stream_id.as_bytes()); - buf.put_u128(check.id); - buf.freeze() -} - -pub fn deser_key(mut bytes: Bytes) -> Result<(StreamId, Check), DeserializationError> { - check_exact_size(&bytes, KEY_LEN)?; - let ordinal = bytes.get_u8(); - if ordinal != KeyType::StreamDeleteOnEmptyCheck as u8 { - return Err(DeserializationError::InvalidOrdinal(ordinal)); - } - let at = TimestampSecs::from_secs(bytes.get_u32()); - let mut stream_id = [0; StreamId::LEN]; - bytes.copy_to_slice(&mut stream_id); - let id = bytes.get_u128(); - Ok((stream_id.into(), Check { at, id })) -} - -pub fn due_key_range(now: TimestampSecs) -> Range { - let start = Bytes::from_static(&[KeyType::StreamDeleteOnEmptyCheck as u8]); - let mut end = BytesMut::with_capacity(5); - end.put_u8(KeyType::StreamDeleteOnEmptyCheck as u8); - end.put_u32(now.as_u32()); - start..super::increment_bytes(end).expect("non-empty") -} - -#[cfg(test)] -mod tests { - use proptest::prelude::*; - - use super::*; - - proptest! { - #[test] - fn roundtrip_check(at in any::(), id in any::(), stream in any::<[u8; StreamId::LEN]>()) { - let check = Check { at: TimestampSecs::from_secs(at), id }; - let stream_id = StreamId::from(stream); - let key = ser_key(stream_id, check); - prop_assert_eq!(deser_key(key.clone()).unwrap(), (stream_id, check)); - prop_assert_eq!(Bytes::from(super::super::Key::try_from(key.clone()).unwrap()), key); - } - } - - #[test] - fn due_range_includes_every_ticket_at_boundary() { - for seconds in [0, 100, u32::MAX] { - let at = TimestampSecs::from_secs(seconds); - let range = due_key_range(at); - for stream_id in [[0; StreamId::LEN], [u8::MAX; StreamId::LEN]].map(StreamId::from) { - for id in [0, u128::MAX] { - assert!(range.contains(&ser_key(stream_id, Check { at, id }))); - if let Some(next) = seconds.checked_add(1) { - assert!(!range.contains(&ser_key( - stream_id, - Check { - at: TimestampSecs::from_secs(next), - id - } - ))); - } - } - } - } - } -} diff --git a/lite/src/backend/kv/stream_doe_deadline.rs b/lite/src/backend/kv/stream_doe_deadline.rs index 8eef06a3..573c5217 100644 --- a/lite/src/backend/kv/stream_doe_deadline.rs +++ b/lite/src/backend/kv/stream_doe_deadline.rs @@ -1,35 +1,48 @@ +use std::{ops::Range, time::Duration}; + use bytes::{Buf, BufMut, Bytes, BytesMut}; -use super::{DeserializationError, KeyType, check_exact_size}; -use crate::{backend::timestamp::TimestampSecs, stream_id::StreamId}; +use super::{DeserializationError, KeyType, check_exact_size, timestamp::TimestampSecs}; +use crate::stream_id::StreamId; -const LEGACY_KEY_LEN: usize = 1 + 4 + StreamId::LEN; -const KEY_LEN: usize = LEGACY_KEY_LEN + 16; -/// Legacy keys are retained only for decoding and migration. New schedules use -/// `stream_doe_state` and `stream_doe_check`. -pub fn ser_key(deadline: TimestampSecs, stream_id: StreamId, schedule_id: Option) -> Bytes { - let key_len = if schedule_id.is_some() { - KEY_LEN - } else { - LEGACY_KEY_LEN - }; - let mut buf = BytesMut::with_capacity(key_len); +const KEY_LEN: usize = 1 + 4 + StreamId::LEN; +const VALUE_LEN: usize = 8; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(in crate::backend) struct Entry { + pub deadline: TimestampSecs, + pub min_age: Duration, +} + +impl Entry { + pub fn last_write_cutoff(self) -> Option { + self.deadline.checked_sub_duration(self.min_age) + } +} + +pub fn ser_key(deadline: TimestampSecs, stream_id: StreamId) -> Bytes { + let mut buf = BytesMut::with_capacity(KEY_LEN); buf.put_u8(KeyType::StreamDeleteOnEmptyDeadline as u8); buf.put_u32(deadline.as_u32()); buf.put_slice(stream_id.as_bytes()); - if let Some(schedule_id) = schedule_id { - buf.put_u128(schedule_id); - } - debug_assert_eq!(buf.len(), key_len, "serialized length mismatch"); + debug_assert_eq!(buf.len(), KEY_LEN, "serialized length mismatch"); buf.freeze() } -pub fn deser_key( - mut bytes: Bytes, -) -> Result<(TimestampSecs, StreamId, Option), DeserializationError> { - if bytes.len() != LEGACY_KEY_LEN { - check_exact_size(&bytes, KEY_LEN)?; - } +pub fn expired_key_range(deadline: TimestampSecs) -> Range { + let start = Bytes::from(vec![KeyType::StreamDeleteOnEmptyDeadline as u8]); + let end = ser_key_range_end(deadline); + start..end +} + +fn ser_key_range_end(deadline: TimestampSecs) -> Bytes { + let max_stream_id = StreamId::from([u8::MAX; StreamId::LEN]); + let end_key = ser_key(deadline, max_stream_id); + super::increment_bytes(BytesMut::from(end_key.as_ref())).expect("non-empty") +} + +pub fn deser_key(mut bytes: Bytes) -> Result<(TimestampSecs, StreamId), DeserializationError> { + check_exact_size(&bytes, KEY_LEN)?; let ordinal = bytes.get_u8(); if ordinal != (KeyType::StreamDeleteOnEmptyDeadline as u8) { return Err(DeserializationError::InvalidOrdinal(ordinal)); @@ -37,20 +50,32 @@ pub fn deser_key( let deadline_secs = bytes.get_u32(); let mut stream_id_bytes = [0u8; StreamId::LEN]; bytes.copy_to_slice(&mut stream_id_bytes); - let schedule_id = bytes.has_remaining().then(|| bytes.get_u128()); Ok(( TimestampSecs::from_secs(deadline_secs), stream_id_bytes.into(), - schedule_id, )) } +pub fn ser_value(min_age: Duration) -> Bytes { + let mut buf = BytesMut::with_capacity(VALUE_LEN); + buf.put_u64(min_age.as_secs()); + debug_assert_eq!(buf.len(), VALUE_LEN, "serialized length mismatch"); + buf.freeze() +} + +pub fn deser_value(mut bytes: Bytes) -> Result { + check_exact_size(&bytes, VALUE_LEN)?; + Ok(Duration::from_secs(bytes.get_u64())) +} + #[cfg(test)] mod tests { + use std::time::Duration; + use proptest::prelude::*; use crate::{ - backend::{kv::stream_doe_deadline, timestamp::TimestampSecs}, + backend::kv::{stream_doe_deadline, timestamp::TimestampSecs}, stream_id::StreamId, }; @@ -59,17 +84,21 @@ mod tests { fn roundtrip_stream_doe_deadline_key( deadline_secs in any::(), stream_id_bytes in any::<[u8; StreamId::LEN]>(), - schedule_id in proptest::option::of(any::()), ) { let deadline = TimestampSecs::from_secs(deadline_secs); let stream_id = StreamId::from(stream_id_bytes); - let bytes = stream_doe_deadline::ser_key(deadline, stream_id, schedule_id); - let (decoded_deadline, decoded_stream_id, decoded_schedule_id) = stream_doe_deadline::deser_key(bytes.clone()).unwrap(); + let bytes = stream_doe_deadline::ser_key(deadline, stream_id); + let (decoded_deadline, decoded_stream_id) = stream_doe_deadline::deser_key(bytes).unwrap(); prop_assert_eq!(deadline, decoded_deadline); prop_assert_eq!(stream_id, decoded_stream_id); - prop_assert_eq!(schedule_id, decoded_schedule_id); - let decoded = super::super::Key::try_from(bytes.clone()).unwrap(); - prop_assert_eq!(bytes, bytes::Bytes::from(decoded)); } } + + #[test] + fn roundtrip_stream_doe_deadline_value() { + let min_age = Duration::from_secs(123); + let bytes = stream_doe_deadline::ser_value(min_age); + let decoded = stream_doe_deadline::deser_value(bytes).unwrap(); + assert_eq!(min_age, decoded); + } } diff --git a/lite/src/backend/kv/stream_doe_state.rs b/lite/src/backend/kv/stream_doe_state.rs deleted file mode 100644 index e1f07b49..00000000 --- a/lite/src/backend/kv/stream_doe_state.rs +++ /dev/null @@ -1,82 +0,0 @@ -use bytes::{Buf, BufMut, Bytes, BytesMut}; - -use super::{DeserializationError, KeyType, check_exact_size}; -use crate::{backend::timestamp::TimestampSecs, stream_id::StreamId}; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Check { - pub at: TimestampSecs, - pub id: u128, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum State { - Scheduled(Check), - /// An observed record has no expiration. Only a trim can remove it. - Parked, -} - -pub fn ser_key(stream_id: StreamId) -> Bytes { - super::ser_stream_id_key(KeyType::StreamDeleteOnEmptyState, stream_id) -} - -pub fn deser_key(bytes: Bytes) -> Result { - super::deser_stream_id_key(KeyType::StreamDeleteOnEmptyState, bytes) -} - -pub fn ser_value(state: State) -> Bytes { - let mut buf = BytesMut::with_capacity(21); - match state { - State::Parked => buf.put_u8(0), - State::Scheduled(Check { at, id }) => { - buf.put_u8(1); - buf.put_u32(at.as_u32()); - buf.put_u128(id); - } - } - buf.freeze() -} - -pub fn deser_value(mut bytes: Bytes) -> Result { - super::check_min_size(&bytes, 1)?; - match bytes.get_u8() { - 0 => { - check_exact_size(&bytes, 0)?; - Ok(State::Parked) - } - 1 => { - check_exact_size(&bytes, 20)?; - Ok(State::Scheduled(Check { - at: TimestampSecs::from_secs(bytes.get_u32()), - id: bytes.get_u128(), - })) - } - ordinal => Err(DeserializationError::InvalidOrdinal(ordinal)), - } -} - -#[cfg(test)] -mod tests { - use proptest::prelude::*; - - use super::*; - - proptest! { - #[test] - fn roundtrip_state(at in any::(), id in any::(), stream in any::<[u8; StreamId::LEN]>()) { - for state in [State::Parked, State::Scheduled(Check { at: TimestampSecs::from_secs(at), id })] { - prop_assert_eq!(deser_value(ser_value(state)).unwrap(), state); - } - let key = ser_key(stream.into()); - prop_assert_eq!(deser_key(key.clone()).unwrap(), StreamId::from(stream)); - prop_assert_eq!(Bytes::from(super::super::Key::try_from(key.clone()).unwrap()), key); - } - } - - #[test] - fn reject_truncated_or_unknown_state() { - for bytes in [&[][..], &[0, 1], &[1], &[2]] { - assert!(deser_value(Bytes::copy_from_slice(bytes)).is_err()); - } - } -} diff --git a/lite/src/backend/kv/stream_meta.rs b/lite/src/backend/kv/stream_meta.rs index f655aaa7..70e22a80 100644 --- a/lite/src/backend/kv/stream_meta.rs +++ b/lite/src/backend/kv/stream_meta.rs @@ -16,7 +16,6 @@ use super::{ DeserializationError, KeyType, check_min_size, deser_json_value, increment_bytes, invalid_value_err, ser_json_value, }; -use crate::backend::resolve_stream_config; const FIELD_SEPARATOR: u8 = b'\0'; @@ -57,12 +56,12 @@ impl TryFrom for StreamMeta { fn try_from(serde: StreamMetaSerde) -> Result { let config = match serde.config { - Some(api_config) => OptionalStreamConfig::try_from(api_config)?, - None => OptionalStreamConfig::default(), + Some(api_config) => OptionalStreamConfig::try_from(api_config)?.into(), + None => StreamConfig::default(), }; Ok(Self { - config: resolve_stream_config(config, OptionalStreamConfig::default()), + config, cipher: serde.cipher, created_at: serde.created_at, deleted_at: serde.deleted_at, @@ -156,7 +155,7 @@ mod tests { use proptest::prelude::*; use s2_common::{ basin::BasinName, - config::{OptionalDeleteOnEmptyConfig, OptionalStreamConfig, StreamConfig}, + config::{OptionalDeleteOnEmptyConfig, OptionalStreamConfig, StorageClass, StreamConfig}, encryption::EncryptionAlgorithm, stream::{StreamName, StreamNamePrefix, StreamNameStartAfter}, }; @@ -168,7 +167,7 @@ mod tests { #[test] fn value_roundtrip_stream_meta() { let config = OptionalStreamConfig { - storage_class: Some("express".into()), + storage_class: Some(StorageClass::Express), delete_on_empty: OptionalDeleteOnEmptyConfig { min_age: Some(Duration::ZERO), }, @@ -229,7 +228,7 @@ mod tests { let decoded = super::deser_value(bytes).unwrap(); let default_config = StreamConfig::default(); - assert_eq!(decoded.config.storage_class.as_deref(), Some("express")); + assert_eq!(decoded.config.storage_class, default_config.storage_class); assert_eq!( decoded.config.retention_policy, default_config.retention_policy diff --git a/lite/src/backend/timestamp.rs b/lite/src/backend/kv/timestamp.rs similarity index 91% rename from lite/src/backend/timestamp.rs rename to lite/src/backend/kv/timestamp.rs index 1455cac7..ae368b8e 100644 --- a/lite/src/backend/timestamp.rs +++ b/lite/src/backend/kv/timestamp.rs @@ -44,14 +44,6 @@ impl TimestampSecs { .map(|secs| Self(secs as u32)) } - pub fn saturating_add_duration(self, dur: Duration) -> Self { - Self( - u64::from(self.0) - .saturating_add(dur.as_secs()) - .min(u64::from(u32::MAX)) as u32, - ) - } - fn from_system_time(time: SystemTime) -> Self { match time.duration_since(UNIX_EPOCH) { Ok(duration) => { diff --git a/lite/src/backend/mod.rs b/lite/src/backend/mod.rs index 99aab0fb..be8db24f 100644 --- a/lite/src/backend/mod.rs +++ b/lite/src/backend/mod.rs @@ -1,23 +1,15 @@ -use s2_common::{ - config::{OptionalStreamConfig, StreamConfig}, - encryption::EncryptionSpec, -}; +use s2_common::encryption::EncryptionSpec; pub mod error; mod basins; pub mod bgtasks; mod core; -mod doe; mod durability_notifier; mod read; mod store; mod streamer; mod streams; -mod timestamp; - -#[cfg(test)] -mod test_util; mod append; mod kv; @@ -33,12 +25,3 @@ pub struct StreamHandle { } pub const FOLLOWER_MAX_LAG: usize = 25; - -fn resolve_stream_config( - config: OptionalStreamConfig, - basin_defaults: OptionalStreamConfig, -) -> StreamConfig { - let mut config = config.merge(basin_defaults); - config.storage_class.get_or_insert_with(|| "express".into()); - config -} diff --git a/lite/src/backend/read.rs b/lite/src/backend/read.rs index 3ff6c5bc..7043409f 100644 --- a/lite/src/backend/read.rs +++ b/lite/src/backend/read.rs @@ -4,7 +4,6 @@ use futures::{Stream, StreamExt as _}; use s2_common::{ basin::BasinName, caps, - config::OptionalStreamConfig, encryption::{EncryptionKey, EncryptionSpec}, read_extent::{EvaluatedReadLimit, ReadLimit, ReadUntil}, record::{Metered, MeteredSize as _, SeqNum, StreamPosition, Timestamp}, @@ -16,7 +15,7 @@ use s2_storage::record::{ use slatedb::config::{DurabilityLevel, ScanOptions}; use tokio::{sync::broadcast, time::Instant}; -use super::{Backend, StreamHandle, core::AutoCreateOn}; +use super::{Backend, StreamHandle}; use crate::{ backend::{ error::{ @@ -29,7 +28,6 @@ use crate::{ }; impl Backend { - /// Open a stream for a check tail. pub async fn open_for_check_tail( &self, basin: &BasinName, @@ -38,29 +36,22 @@ impl Backend { self.stream_handle_with_auto_create::( basin, stream, - AutoCreateOn::Read, - OptionalStreamConfig::default(), + |config| config.create_stream_on_read, |_| Ok(EncryptionSpec::Plain), ) .await } - /// Open a stream for a read or read session. - /// - /// `stream_config` is applied if the stream is created on read. Unset fields inherit the - /// basin's default stream configuration. Ignored if the stream already exists. pub async fn open_for_read( &self, basin: &BasinName, stream: &StreamName, encryption_key: Option, - stream_config: OptionalStreamConfig, ) -> Result { self.stream_handle_with_auto_create::( basin, stream, - AutoCreateOn::Read, - stream_config, + |config| config.create_stream_on_read, |cipher| Ok(EncryptionSpec::resolve(cipher, encryption_key)?), ) .await @@ -106,10 +97,6 @@ async fn read_session( start: ReadStart, end: ReadEnd, ) -> Result> + 'static, ReadError> { - // An exhausted limit completes even when the requested start is unwritten. - if end.limit.remaining(0, 0) == EvaluatedReadLimit::Exhausted { - return Ok(futures::stream::empty().left_stream()); - } let stream_id = client.stream_id(); let tail = client.check_tail().await?; let mut state = ReadSessionState { @@ -253,7 +240,7 @@ async fn read_session( } } }; - Ok(session.right_stream()) + Ok(session) } async fn read_start_seq_num( @@ -280,9 +267,13 @@ async fn read_start_seq_num( return Err(UnwrittenError(tail).into()); } } - // Resolve to a sequence number before deciding whether the read starts at the tail, so a - // timestamp start that resolves to the tail is treated like a sequence number start there. - let start_seq_num = match read_pos { + if let ReadPosition::SeqNum(start_seq_num) = read_pos + && start_seq_num == tail.seq_num + && !end.may_follow() + { + return Err(UnwrittenError(tail).into()); + } + Ok(match read_pos { ReadPosition::SeqNum(start_seq_num) => start_seq_num, ReadPosition::Timestamp(start_timestamp) => { resolve_timestamp(db, stream_id, start_timestamp) @@ -290,11 +281,7 @@ async fn read_start_seq_num( .unwrap_or(tail) .seq_num } - }; - if start_seq_num == tail.seq_num && !end.may_follow() { - return Err(UnwrittenError(tail).into()); - } - Ok(start_seq_num) + }) } async fn resolve_timestamp( @@ -434,9 +421,7 @@ mod tests { use super::*; use crate::{ - backend::{ - FOLLOWER_MAX_LAG, kv, streamer::DORMANT_TIMEOUT, test_util::DbWriteTestExt as _, - }, + backend::{FOLLOWER_MAX_LAG, kv, streamer::DORMANT_TIMEOUT}, stream_id::StreamId, }; @@ -511,8 +496,8 @@ mod tests { ), kv::stream_record_timestamp::ser_value(), ) - .assert_durable() - .await; + .await + .unwrap(); backend .db .put( @@ -525,8 +510,8 @@ mod tests { ), kv::stream_record_timestamp::ser_value(), ) - .assert_durable() - .await; + .await + .unwrap(); // Should find record in stream_a let result = resolve_timestamp(&backend.db, stream_a, 500).await.unwrap(); @@ -577,7 +562,7 @@ mod tests { let input = append_input(Record::try_from_parts(vec![], bytes::Bytes::from("x")).unwrap()); let ack = backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_append(&basin, &stream, None) .await .unwrap() .append(input) @@ -588,7 +573,7 @@ mod tests { let stream_id = StreamId::new(&basin, &stream); let mut batch = WriteBatch::new(); batch.delete(kv::stream_record_data::ser_key(stream_id, ack.start)); - backend.db.write(batch).assert_durable().await; + backend.db.write(batch).await.unwrap(); let start = ReadStart { from: ReadFrom::SeqNum(0), @@ -600,7 +585,7 @@ mod tests { wait: None, }; let session = backend - .open_for_read(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_read(&basin, &stream, None) .await .unwrap() .read(start, end) @@ -657,7 +642,7 @@ mod tests { }; let session = backend - .open_for_read(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_read(&basin, &stream, None) .await .unwrap() .read(start, end) @@ -731,7 +716,7 @@ mod tests { let initial_input = append_input(Record::try_from_parts(vec![], bytes::Bytes::from("initial")).unwrap()); backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_append(&basin, &stream, None) .await .unwrap() .append(initial_input) @@ -751,7 +736,7 @@ mod tests { }; let session = backend - .open_for_read(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_read(&basin, &stream, None) .await .unwrap() .read(start, end) @@ -791,7 +776,7 @@ mod tests { let follow_input = append_input(Record::try_from_parts(vec![], bytes::Bytes::from("follow-1")).unwrap()); backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_append(&basin, &stream, None) .await .unwrap() .append(follow_input) @@ -896,7 +881,7 @@ mod tests { wait: Some(wait), }; let session = backend - .open_for_read(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_read(&basin, &stream, None) .await .unwrap() .read(start, end) @@ -921,7 +906,7 @@ mod tests { Record::try_from_parts(vec![], bytes::Bytes::from(format!("lagged-{i}"))).unwrap(), ); let ack = backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_append(&basin, &stream, None) .await .unwrap() .append(input) @@ -930,7 +915,7 @@ mod tests { delete_batch.delete(kv::stream_record_data::ser_key(stream_id, ack.start)); } - backend.db.write(delete_batch).assert_durable().await; + backend.db.write(delete_batch).await.unwrap(); tokio::time::advance(wait + Duration::from_secs(1)).await; tokio::task::yield_now().await; @@ -975,7 +960,7 @@ mod tests { let initial_input = append_input(Record::try_from_parts(vec![], bytes::Bytes::from("initial")).unwrap()); backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_append(&basin, &stream, None) .await .unwrap() .append(initial_input) @@ -992,7 +977,7 @@ mod tests { wait: None, }; let session = backend - .open_for_read(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_read(&basin, &stream, None) .await .unwrap() .read(start, end) @@ -1022,7 +1007,7 @@ mod tests { let follow_input = append_input(Record::try_from_parts(vec![], bytes::Bytes::from("follow-1")).unwrap()); backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) + .open_for_append(&basin, &stream, None) .await .unwrap() .append(follow_input) diff --git a/lite/src/backend/store.rs b/lite/src/backend/store.rs index 6b26bcdb..791bc561 100644 --- a/lite/src/backend/store.rs +++ b/lite/src/backend/store.rs @@ -1,6 +1,6 @@ use bytes::Bytes; use slatedb::{ - DbSnapshot, DbTransaction, KeyValue, + DbTransaction, config::{DurabilityLevel, ReadOptions}, }; @@ -19,15 +19,6 @@ impl Backend { &self, key: K, deser: impl FnOnce(Bytes) -> Result, - ) -> Result, StorageError> { - self.db_get_with(key, |entry| deser(entry.value)).await - } - - /// Read a remotely durable row, including its sequence and timestamps. - pub(super) async fn db_get_with + Send, V>( - &self, - key: K, - deser: impl FnOnce(KeyValue) -> Result, ) -> Result, StorageError> { let read_opts = ReadOptions { durability_filter: DurabilityLevel::Remote, @@ -35,7 +26,7 @@ impl Backend { }; let value = self .db - .get_key_value_with_options(key, &read_opts) + .get_with_options(key, &read_opts) .await? .map(deser) .transpose()?; @@ -43,40 +34,11 @@ impl Backend { } } -pub(super) async fn db_snapshot_get_with + Send, V>( - snapshot: &DbSnapshot, - key: K, - deser: impl FnOnce(KeyValue) -> Result, -) -> Result, StorageError> { - Ok(snapshot.get_key_value(key).await?.map(deser).transpose()?) -} - pub(super) async fn db_txn_get + Send, V>( txn: &DbTransaction, key: K, deser: impl FnOnce(Bytes) -> Result, ) -> Result, StorageError> { - db_txn_get_with(txn, key, |entry| deser(entry.value)).await -} - -/// Read a transaction row, which may not yet be durable. -pub(super) async fn db_txn_get_with + Send, V>( - txn: &DbTransaction, - key: K, - deser: impl FnOnce(KeyValue) -> Result, -) -> Result, StorageError> { - let value = txn.get_key_value(key).await?.map(deser).transpose()?; + let value = txn.get(key).await?.map(deser).transpose()?; Ok(value) } - -/// Commit metadata changes and wait until remote reads can observe them. -/// Return the committed sequence number, or `None` if there were no writes. -pub(super) async fn db_txn_commit_durable( - txn: DbTransaction, -) -> Result, slatedb::Error> { - let Some(handle) = txn.commit().await? else { - return Ok(None); - }; - handle.await_durable().await?; - Ok(Some(handle.seqnum())) -} diff --git a/lite/src/backend/streamer.rs b/lite/src/backend/streamer.rs index 1dd567e8..d8acb6a3 100644 --- a/lite/src/backend/streamer.rs +++ b/lite/src/backend/streamer.rs @@ -28,7 +28,7 @@ use s2_storage::record::{ }; use slatedb::{ IterationOrder, WriteBatch, - config::{PutOptions, ScanOptions, Ttl}, + config::{PutOptions, ScanOptions, Ttl, WriteOptions}, }; use tokio::{ sync::{Semaphore, SemaphorePermit, broadcast, mpsc, oneshot}, @@ -40,21 +40,27 @@ use crate::{ backend::{ append, bgtasks::BgtaskTrigger, - doe, durability_notifier::DurabilityNotifier, error::{ AppendConditionFailedError, AppendErrorInternal, AppendTimestampRequiredError, DeleteStreamError, MaxSeqNumError, RequestDroppedError, StorageError, - StreamerMissingInActionError, + StreamDeletionPendingError, StreamerMissingInActionError, }, kv, - timestamp::TimestampSecs, }, metrics, stream_id::StreamId, }; pub(super) const DORMANT_TIMEOUT: Duration = Duration::from_secs(60); +// Rate-limit delete-on-empty scheduling and pad deadlines to cover the period. +const DOE_DEADLINE_REFRESH_PERIOD: Duration = Duration::from_secs(600); + +pub(super) fn doe_arm_delay(base_delay: Duration, min_age: Duration) -> Duration { + base_delay + .saturating_add(min_age) + .saturating_add(DOE_DEADLINE_REFRESH_PERIOD) +} #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) struct StreamerGenerationId(u64); @@ -74,6 +80,7 @@ struct InFlightAppend { struct DbSubmitAppendOptions { retention: RetentionPolicy, + doe_deadline: Option, fencing_token: Option, trim_point: Option>, } @@ -203,14 +210,10 @@ pub(super) struct Spawner { pub generation_id: StreamerGenerationId, pub db: slatedb::Db, pub stream_id: StreamId, - /// Database commit sequence that created the stream's ID mapping. - /// Stable across streamer restarts; changes when the stream is recreated. - pub stream_creation_seq: u64, pub config: StreamConfig, - pub config_seq: u64, pub cipher: Option, pub tail_pos: StreamPosition, - pub last_tail_write_timestamp: TimestampSecs, + pub last_tail_write_timestamp: kv::timestamp::TimestampSecs, pub fencing_token: FencingToken, pub trim_point: RangeTo, pub append_inflight_bytes_sema: Arc, @@ -227,9 +230,7 @@ impl Spawner { generation_id, db, stream_id, - stream_creation_seq, config, - config_seq, cipher, tail_pos, last_tail_write_timestamp, @@ -245,10 +246,8 @@ impl Spawner { let streamer = Streamer { db, stream_id, - stream_creation_seq, msg_tx: msg_tx.clone(), config, - config_seq, last_tail_write_timestamp, fencing_token: CommandState { state: fencing_token, @@ -258,6 +257,7 @@ impl Spawner { state: trim_point, applied_point: ..tail_pos.seq_num, }, + last_doe_deadline_at: None, db_writes_pending: VecDeque::new(), db_durability_subscription: 0, inflight_appends: VecDeque::new(), @@ -306,13 +306,12 @@ impl CommandState { struct Streamer { db: slatedb::Db, stream_id: StreamId, - stream_creation_seq: u64, msg_tx: mpsc::UnboundedSender, config: StreamConfig, - config_seq: u64, - last_tail_write_timestamp: TimestampSecs, + last_tail_write_timestamp: kv::timestamp::TimestampSecs, fencing_token: CommandState, trim_point: CommandState>, + last_doe_deadline_at: Option, db_writes_pending: VecDeque>>, db_durability_subscription: u64, inflight_appends: VecDeque, @@ -401,10 +400,7 @@ impl Streamer { return; }; let sequenced_records = if self.trim_point.state.end == SeqNum::MAX { - Err(AppendErrorInternal::StreamDeletionPending { - // The terminal trim must be durable before reporting deletion pending. - durability_dependency: self.trim_point.applied_point, - }) + Err(StreamDeletionPendingError.into()) } else { self.sequence_records(input) }; @@ -426,6 +422,7 @@ impl Streamer { let seq_num_range = first_pos.seq_num..next_pos.seq_num; let opts = DbSubmitAppendOptions { retention: self.config.retention_policy, + doe_deadline: self.doe_deadline_maybe(), fencing_token: self .fencing_token .is_applied_in(&seq_num_range) @@ -440,7 +437,7 @@ impl Streamer { .boxed(), ); self.pending_appends.accept(ticket, first_pos..next_pos); - self.last_tail_write_timestamp = TimestampSecs::now(); + self.last_tail_write_timestamp = kv::timestamp::TimestampSecs::now(); } Err(e) => { self.pending_appends.reject(ticket, e, self.stable_pos); @@ -455,41 +452,30 @@ impl Streamer { ) { match condition { TerminalTrimCondition::Always => { - self.ensure_terminal_trim(reply_tx); + self.append_terminal_trim(reply_tx); } - TerminalTrimCondition::DeleteOnEmpty { - expected_stream_creation_seq, - expected_config_seq, - } => { - if self.stream_creation_seq != expected_stream_creation_seq { - let _ = reply_tx.send(Ok(TerminalTrimOutcome::Obsolete)); - } else if self.trim_point.state.end == SeqNum::MAX { - self.ensure_terminal_trim(reply_tx); - } else if self.config_seq != expected_config_seq { - // The worker may have observed a configuration commit before - // its notification reached this actor (or vice versa). Do not - // defer using an age that may have just been decreased. - let _ = reply_tx.send(Ok(TerminalTrimOutcome::RetryAt(TimestampSecs::after( - doe::RETRY_INTERVAL, - )))); - } else if self.config.delete_on_empty.min_age().is_none() { - let _ = reply_tx.send(Ok(TerminalTrimOutcome::Obsolete)); - } else { - let db = self.db.clone(); - let stream_id = self.stream_id; - let stable_pos_snapshot = self.stable_pos; - let config_seq_snapshot = self.config_seq; - let msg_tx = self.msg_tx.clone(); - tokio::spawn(async move { - let records = stream_record_presence(&db, stream_id).await; - let _ = msg_tx.send(Message::DeleteOnEmptyCheckResult { - stable_pos_snapshot, - config_seq_snapshot, - records, - reply_tx, - }); - }); + TerminalTrimCondition::DeleteOnEmpty { last_write_cutoff } => { + if self.last_tail_write_timestamp > last_write_cutoff + || self.next_assignable_pos().seq_num != self.stable_pos.seq_num + || self.config.delete_on_empty.min_age().is_none() + { + let _ = reply_tx.send(Ok(TerminalTrimOutcome::Ineligible)); + return; } + + let db = self.db.clone(); + let stream_id = self.stream_id; + let stable_pos_snapshot = self.stable_pos; + let msg_tx = self.msg_tx.clone(); + tokio::spawn(async move { + let has_records = stream_has_records(&db, stream_id).await; + let _ = msg_tx.send(Message::DeleteOnEmptyCheckResult { + stable_pos_snapshot, + last_write_cutoff, + has_records, + reply_tx, + }); + }); } } } @@ -497,66 +483,34 @@ impl Streamer { fn handle_doe_check_result( &mut self, stable_pos_snapshot: StreamPosition, - config_seq_snapshot: u64, - records: Result, + last_write_cutoff: kv::timestamp::TimestampSecs, + has_records: Result, reply_tx: oneshot::Sender>, ) { - let records = match records { - Ok(records) => records, + match has_records { + Ok(true) => { + let _ = reply_tx.send(Ok(TerminalTrimOutcome::Ineligible)); + } + Ok(false) => { + if self.trim_point.state.end == SeqNum::MAX { + let _ = reply_tx.send(Ok(TerminalTrimOutcome::DeletionPending)); + } else if self.stable_pos != stable_pos_snapshot + || self.next_assignable_pos() != stable_pos_snapshot + || self.last_tail_write_timestamp > last_write_cutoff + || self.config.delete_on_empty.min_age().is_none() + { + let _ = reply_tx.send(Ok(TerminalTrimOutcome::Ineligible)); + } else { + self.append_terminal_trim(reply_tx); + } + } Err(err) => { let _ = reply_tx.send(Err(err.into())); - return; } - }; - if self.trim_point.state.end == SeqNum::MAX { - self.ensure_terminal_trim(reply_tx); - return; } - if self.config_seq != config_seq_snapshot { - let _ = reply_tx.send(Ok(TerminalTrimOutcome::RetryAt(TimestampSecs::after( - doe::RETRY_INTERVAL, - )))); - return; - } - let Some(min_age) = self.config.delete_on_empty.min_age() else { - let _ = reply_tx.send(Ok(TerminalTrimOutcome::Obsolete)); - return; - }; - let outcome = match records { - // Appends cannot remove an observed record, so these bounds remain - // useful even when the tail advances. The scheduler's revision check - // protects against a trim removing the record before we finish. - RecordPresence::ExpiresAt(at) => self.doe_retry_at(at), - RecordPresence::Unbounded => TerminalTrimOutcome::Parked, - RecordPresence::Empty => { - let old_enough = TimestampSecs::now() - .checked_sub_duration(min_age) - .is_some_and(|cutoff| self.last_tail_write_timestamp <= cutoff); - if self.stable_pos == stable_pos_snapshot - && self.next_assignable_pos() == stable_pos_snapshot - && old_enough - { - self.ensure_terminal_trim(reply_tx); - return; - } - self.doe_retry_at(TimestampSecs::ZERO) - } - }; - let _ = reply_tx.send(Ok(outcome)); } - fn doe_retry_at(&self, earliest_empty: TimestampSecs) -> TerminalTrimOutcome { - let age_at = self - .last_tail_write_timestamp - .saturating_add_duration(self.config.delete_on_empty.min_age().unwrap_or_default()); - TerminalTrimOutcome::RetryAt( - TimestampSecs::after(doe::RETRY_INTERVAL) - .max(age_at) - .max(earliest_empty), - ) - } - - fn ensure_terminal_trim( + fn append_terminal_trim( &mut self, reply_tx: oneshot::Sender>, ) { @@ -570,7 +524,7 @@ impl Streamer { tokio::spawn(async move { let result = match append_reply_rx.await { Ok(Ok(_)) => Ok(TerminalTrimOutcome::DeletionPending), - Ok(Err(AppendErrorInternal::StreamDeletionPending { .. })) => { + Ok(Err(AppendErrorInternal::StreamDeletionPending(_))) => { Ok(TerminalTrimOutcome::DeletionPending) } Ok(Err(AppendErrorInternal::Storage(e))) => Err(DeleteStreamError::Storage(e)), @@ -589,12 +543,31 @@ impl Streamer { Ok(Err(AppendErrorInternal::MaxSeqNum(_))) => { unreachable!("terminal append is plaintext command record") } - Err(_) => Err(RequestDroppedError.into()), + Err(_) => Err(DeleteStreamError::StreamerMissingInActionError( + StreamerMissingInActionError, + )), }; let _ = reply_tx.send(result); }); } + fn doe_deadline_maybe(&mut self) -> Option { + let retention_age = self.config.retention_policy.age()?; + let min_age = self.config.delete_on_empty.min_age()?; + let now = Instant::now(); + if self + .last_doe_deadline_at + .is_none_or(|t| now.duration_since(t) >= DOE_DEADLINE_REFRESH_PERIOD) + { + self.last_doe_deadline_at = Some(now); + let deadline = + kv::timestamp::TimestampSecs::after(doe_arm_delay(retention_age, min_age)); + Some(kv::stream_doe_deadline::Entry { deadline, min_age }) + } else { + None + } + } + fn subscribe_durability(&mut self) { if let Some(inflight_append) = self .inflight_appends @@ -686,14 +659,14 @@ impl Streamer { } Message::DeleteOnEmptyCheckResult { stable_pos_snapshot, - config_seq_snapshot, - records, + last_write_cutoff, + has_records, reply_tx, } => { self.handle_doe_check_result( stable_pos_snapshot, - config_seq_snapshot, - records, + last_write_cutoff, + has_records, reply_tx, ); } @@ -711,11 +684,8 @@ impl Streamer { Message::CheckTail { reply_tx } => { let _ = reply_tx.send(self.stable_pos); } - Message::Reconfigure { seq, config } => { - if seq > self.config_seq { - self.config = config; - self.config_seq = seq; - } + Message::Reconfigure { config } => { + self.config = config; } Message::DurabilityStatus(status) => { match status { @@ -736,12 +706,7 @@ impl Streamer { } } _ = dormancy.as_mut() => { - // Cancelled requests can still have writes become durable. Keep - // their assigned positions until a new streamer can recover them. - if self.db_writes_pending.is_empty() - && self.inflight_appends.is_empty() - && self.lease_state.close_if_idle() - { + if self.lease_state.close_if_idle() { break; } } @@ -763,8 +728,8 @@ enum Message { }, DeleteOnEmptyCheckResult { stable_pos_snapshot: StreamPosition, - config_seq_snapshot: u64, - records: Result, + last_write_cutoff: kv::timestamp::TimestampSecs, + has_records: Result, reply_tx: oneshot::Sender>, }, Follow { @@ -777,7 +742,6 @@ enum Message { reply_tx: oneshot::Sender, }, Reconfigure { - seq: u64, config: StreamConfig, }, DurabilityStatus(Result), @@ -786,19 +750,14 @@ enum Message { pub(super) enum TerminalTrimCondition { Always, DeleteOnEmpty { - expected_stream_creation_seq: u64, - expected_config_seq: u64, + last_write_cutoff: kv::timestamp::TimestampSecs, }, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum TerminalTrimOutcome { - /// Deletion is durably pending. DeletionPending, - RetryAt(TimestampSecs), - Parked, - /// DOE is disabled or this request belongs to an earlier incarnation. - Obsolete, + Ineligible, } #[derive(Debug, Clone)] @@ -878,10 +837,8 @@ impl StreamerClient { }) } - pub(super) fn advise_reconfig(&self, seq: u64, config: StreamConfig) -> bool { - self.msg_tx - .send(Message::Reconfigure { seq, config }) - .is_ok() + pub(super) fn advise_reconfig(&self, config: StreamConfig) -> bool { + self.msg_tx.send(Message::Reconfigure { config }).is_ok() } async fn terminal_trim( @@ -897,7 +854,9 @@ impl StreamerClient { .map_err(|_| { DeleteStreamError::StreamerMissingInActionError(StreamerMissingInActionError) })?; - reply_rx.await.map_err(|_| RequestDroppedError)? + reply_rx.await.map_err(|_| { + DeleteStreamError::StreamerMissingInActionError(StreamerMissingInActionError) + })? } } @@ -986,17 +945,7 @@ pub fn next_pos(records: &[Metered]) -> StreamPosition { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum RecordPresence { - Empty, - ExpiresAt(TimestampSecs), - Unbounded, -} - -async fn stream_record_presence( - db: &slatedb::Db, - stream_id: StreamId, -) -> Result { +async fn stream_has_records(db: &slatedb::Db, stream_id: StreamId) -> Result { let prefix = kv::stream_record_timestamp::ser_key_prefix(stream_id); let scan_opts = ScanOptions::default().with_order(IterationOrder::Descending); let mut it = db.scan_prefix_with_options(prefix, .., &scan_opts).await?; @@ -1005,20 +954,11 @@ async fn stream_record_presence( .map(|duration| i64::try_from(duration.as_millis()).unwrap_or(i64::MAX)) .unwrap_or(0); while let Some(kv) = it.next().await? { - match kv.expire_ts { - None => return Ok(RecordPresence::Unbounded), - Some(expire_ts) if expire_ts > now_millis => { - // One live record bounds when the stream can become empty. - // Use its stored TTL, since retention changes are not retroactive. - // Round up so the check does not run just before expiration. - return Ok(RecordPresence::ExpiresAt(TimestampSecs::from_millis( - expire_ts.saturating_add(999), - ))); - } - _ => (), + if kv.expire_ts.is_none_or(|expire_ts| expire_ts > now_millis) { + return Ok(true); } } - Ok(RecordPresence::Empty) + Ok(false) } fn sequenced_records( @@ -1073,46 +1013,56 @@ async fn db_submit_append( records: Vec>, DbSubmitAppendOptions { retention, + doe_deadline, fencing_token, trim_point, }: DbSubmitAppendOptions, ) -> Result { let ttl = match retention { - RetentionPolicy::Age(age) => Ttl::ExpireAfterMillis(age.as_millis() as u64), + RetentionPolicy::Age(age) => Ttl::ExpireAfter(age.as_millis() as u64), RetentionPolicy::Infinite() => Ttl::NoExpiry, }; let ttl_put_opts = PutOptions { ttl }; let mut wb = WriteBatch::new(); for (position, record) in records.iter().map(|msr| msr.parts()) { - wb.put_bytes_with_options( + wb.put_with_options( kv::stream_record_data::ser_key(stream_id, position), kv::stream_record_data::ser_value(record), &ttl_put_opts, ); - wb.put_bytes_with_options( + wb.put_with_options( kv::stream_record_timestamp::ser_key(stream_id, position), kv::stream_record_timestamp::ser_value(), &ttl_put_opts, ); } if let Some(fencing_token) = fencing_token { - wb.put_bytes( + wb.put( kv::stream_fencing_token::ser_key(stream_id), kv::stream_fencing_token::ser_value(&fencing_token), ); } if let Some(trim_point) = trim_point.and_then(|tp| NonZeroSeqNum::new(tp.end)) { - wb.put_bytes( + wb.put( kv::stream_trim_point::ser_key(stream_id), kv::stream_trim_point::ser_value(..trim_point), ); } - wb.put_bytes( + if let Some(doe_deadline) = doe_deadline { + wb.put( + kv::stream_doe_deadline::ser_key(doe_deadline.deadline, stream_id), + kv::stream_doe_deadline::ser_value(doe_deadline.min_age), + ); + } + wb.put( kv::stream_tail_position::ser_key(stream_id), kv::stream_tail_position::ser_value(next_pos(&records)), ); - // The durability notifier tracks this sequence and acknowledges the append after flush. - let write_handle = db.write(wb).await?; + let write_opts = WriteOptions { + await_durable: false, + ..Default::default() + }; + let write_handle = db.write_with_options(wb, &write_opts).await?; Ok(InFlightAppend { db_seq: write_handle.seqnum(), records, @@ -1459,27 +1409,9 @@ mod tests { } } - async fn make_pending_append_durable(streamer: &mut Streamer) { - let submitted = streamer - .db_writes_pending - .pop_front() - .unwrap() - .await - .unwrap(); - let seq = submitted.db_seq; - streamer.inflight_appends.push_back(submitted); - streamer.db.flush().await.unwrap(); - streamer.on_db_durable_seq_advanced(seq); - } - async fn test_streamer() -> Streamer { - test_streamer_with_settings(Default::default()).await - } - - async fn test_streamer_with_settings(settings: slatedb::config::Settings) -> Streamer { let object_store = Arc::new(InMemory::new()); let db = slatedb::Db::builder("/test", object_store) - .with_settings(settings) .build() .await .expect("db"); @@ -1489,11 +1421,9 @@ mod tests { Streamer { db: db.clone(), stream_id: [3u8; StreamId::LEN].into(), - stream_creation_seq: 0, msg_tx, config: StreamConfig::default(), - config_seq: 0, - last_tail_write_timestamp: TimestampSecs::ZERO, + last_tail_write_timestamp: kv::timestamp::TimestampSecs::ZERO, fencing_token: CommandState { state: FencingToken::default(), applied_point: ..SeqNum::MIN, @@ -1502,6 +1432,7 @@ mod tests { state: ..SeqNum::MIN, applied_point: ..SeqNum::MIN, }, + last_doe_deadline_at: None, db_writes_pending: VecDeque::new(), db_durability_subscription: 0, inflight_appends: VecDeque::new(), @@ -1514,62 +1445,6 @@ mod tests { } } - #[tokio::test] - async fn stale_config_notifications_cannot_restore_old_retention() { - let mut streamer = test_streamer().await; - let db = streamer.db.clone(); - let stream_id = streamer.stream_id; - let (msg_tx, msg_rx) = mpsc::unbounded_channel(); - streamer.msg_tx = msg_tx.clone(); - streamer.config.retention_policy = RetentionPolicy::Age(Duration::from_secs(1)); - let task = tokio::spawn(streamer.run(msg_rx)); - msg_tx - .send(Message::Reconfigure { - seq: 20, - config: StreamConfig { - retention_policy: RetentionPolicy::Infinite(), - ..Default::default() - }, - }) - .unwrap(); - let old = StreamConfig { - retention_policy: RetentionPolicy::Age(Duration::from_secs(1)), - ..Default::default() - }; - msg_tx - .send(Message::Reconfigure { - seq: 10, - config: old, - }) - .unwrap(); - let (reply_tx, reply_rx) = oneshot::channel(); - msg_tx - .send(Message::Append { - input: append_input(b"must not expire"), - session: None, - reply_tx, - append_type: AppendType::Regular, - }) - .unwrap(); - let ack = tokio::time::timeout(Duration::from_secs(5), reply_rx) - .await - .unwrap() - .unwrap() - .unwrap(); - let entry = db - .get_key_value(kv::stream_record_data::ser_key(stream_id, ack.start)) - .await - .unwrap() - .unwrap(); - assert!( - entry.expire_ts.is_none(), - "late config notification restored stale retention" - ); - task.abort(); - assert!(task.await.unwrap_err().is_cancelled()); - db.close().await.unwrap(); - } - #[test] fn lease_state_closes_when_idle_and_rejects_new_leases() { let (streamer_lease_state, client_lease_state) = StreamerLeaseState::new(); @@ -1614,304 +1489,60 @@ mod tests { } #[tokio::test] - async fn terminal_trim_and_rejections_wait_for_durability() { - let mut streamer = test_streamer_with_settings(slatedb::config::Settings { - flush_interval: None, - ..Default::default() - }) - .await; - let mut replies = Vec::new(); - for _ in 0..2 { - let (tx, rx) = oneshot::channel(); - streamer.handle_terminal_trim(TerminalTrimCondition::Always, tx); - replies.push(rx); - } - // An empty-stream check can also finish after another deletion request starts. - let (tx, rx) = oneshot::channel(); - streamer.handle_doe_check_result( - StreamPosition::MIN, - streamer.config_seq, - Ok(RecordPresence::Empty), - tx, - ); - replies.push(rx); + async fn append_during_terminal_trim_returns_stream_deletion_pending() { + let mut streamer = test_streamer().await; + streamer.trim_point = CommandState { + state: ..SeqNum::MAX, + applied_point: ..1, + }; + let (msg_tx, msg_rx) = mpsc::unbounded_channel(); + let run_handle = tokio::spawn(streamer.run(msg_rx)); - let (tx, mut append_reply) = oneshot::channel(); - streamer.handle_append(append_input(b"late"), None, tx, AppendType::Regular); - assert_eq!(streamer.db_writes_pending.len(), 1); - tokio::task::yield_now().await; - assert!(matches!( - append_reply.try_recv(), - Err(oneshot::error::TryRecvError::Empty) - )); - for reply in &mut replies { - assert!( - matches!(reply.try_recv(), Err(oneshot::error::TryRecvError::Empty)), - "stream deletion must wait even when the original trim has not been submitted" - ); - } + let (reply_tx, reply_rx) = oneshot::channel(); + msg_tx + .send(Message::Append { + input: append_input(b"late"), + session: None, + reply_tx, + append_type: AppendType::Regular, + }) + .expect("streamer should accept append message"); - let submitted = streamer - .db_writes_pending - .pop_front() - .unwrap() + let err = reply_rx .await - .unwrap(); - let db_seq = submitted.db_seq; - streamer.inflight_appends.push_back(submitted); - tokio::task::yield_now().await; + .expect("streamer should reply") + .expect_err("append should be rejected"); + let AppendErrorInternal::StreamDeletionPending(_) = err else { + panic!("expected stream deletion pending"); + }; + + run_handle.abort(); + } + + #[tokio::test] + async fn delete_on_empty_terminal_trim_skips_pending_append() { + let mut streamer = test_streamer().await; + let (append_tx, mut append_rx) = oneshot::channel(); + streamer.handle_append(append_input(b"live"), None, append_tx, AppendType::Regular); + assert_eq!(streamer.db_writes_pending.len(), 1); assert!(matches!( - append_reply.try_recv(), - Err(oneshot::error::TryRecvError::Empty) + append_rx.try_recv(), + Err(tokio::sync::oneshot::error::TryRecvError::Empty) )); - for reply in &mut replies { - assert!( - matches!(reply.try_recv(), Err(oneshot::error::TryRecvError::Empty)), - "a committed but unflushed trim must not acknowledge stream deletion" - ); - } - streamer.db.flush().await.unwrap(); - streamer.on_db_durable_seq_advanced(db_seq); - assert!(matches!( - append_reply.await.unwrap(), - Err(AppendErrorInternal::StreamDeletionPending { .. }) - )); - for reply in replies { - assert_eq!( - reply.await.unwrap().unwrap(), - TerminalTrimOutcome::DeletionPending - ); - } - streamer.db.close().await.unwrap(); - } - #[rstest::rstest] - #[case::pending_before_scan(false, false)] - #[case::pending_after_scan(true, false)] - #[case::durable_after_scan(true, true)] - #[tokio::test] - async fn delete_on_empty_uses_nonempty_observation_despite_appends( - #[case] append_after_scan: bool, - #[case] durable: bool, - #[values(false, true)] infinite: bool, - ) { - let mut streamer = test_streamer().await; - streamer.config.delete_on_empty.min_age = Duration::from_secs(60); - streamer.config.retention_policy = if infinite { - RetentionPolicy::Infinite() - } else { - RetentionPolicy::Age(Duration::from_secs(3600)) - }; - let (seed_tx, seed_rx) = oneshot::channel(); - streamer.handle_append( - append_input(b"observed record"), - None, - seed_tx, - AppendType::Regular, - ); - make_pending_append_durable(&mut streamer).await; - seed_rx.await.unwrap().unwrap(); - - let (msg_tx, mut msg_rx) = mpsc::unbounded_channel(); - streamer.msg_tx = msg_tx; - let (append_tx, append_rx) = oneshot::channel(); - // Keep the sender until the selected point in the asynchronous check. - let mut append_tx = Some(append_tx); - if !append_after_scan { - streamer.handle_append( - append_input(b"pending before scan"), - None, - append_tx.take().unwrap(), - AppendType::Regular, - ); - } - let (reply_tx, reply_rx) = oneshot::channel(); + let (trim_tx, trim_rx) = oneshot::channel(); streamer.handle_terminal_trim( TerminalTrimCondition::DeleteOnEmpty { - expected_stream_creation_seq: streamer.stream_creation_seq, - expected_config_seq: streamer.config_seq, + last_write_cutoff: kv::timestamp::TimestampSecs::MAX, }, - reply_tx, + trim_tx, ); - let Message::DeleteOnEmptyCheckResult { - stable_pos_snapshot, - config_seq_snapshot, - records, - reply_tx, - } = msg_rx.recv().await.unwrap() - else { - panic!("expected record check even with a pending append"); - }; - if append_after_scan { - streamer.handle_append( - append_input(b"arrived after scan"), - None, - append_tx.take().unwrap(), - AppendType::Regular, - ); - } - if durable { - make_pending_append_durable(&mut streamer).await; - append_rx.await.unwrap().unwrap(); - } - streamer.handle_doe_check_result( - stable_pos_snapshot, - config_seq_snapshot, - records, - reply_tx, - ); - assert_eq!(streamer.db_writes_pending.len(), usize::from(!durable)); - assert_ne!(streamer.trim_point.state.end, SeqNum::MAX); - let outcome = reply_rx.await.unwrap().unwrap(); - if infinite { - assert_eq!(outcome, TerminalTrimOutcome::Parked); - } else { - assert!(matches!( - outcome, - TerminalTrimOutcome::RetryAt(at) - if at > TimestampSecs::after(Duration::from_secs(3500)) - )); - } - streamer.db.close().await.unwrap(); - } - #[rstest::rstest] - #[case::pending_before_scan(false, false)] - #[case::pending_after_scan(true, false)] - #[case::durable_after_scan(true, true)] - #[tokio::test] - async fn delete_on_empty_rechecks_appends_after_empty_scan( - #[case] append_after_scan: bool, - #[case] durable: bool, - ) { - let mut streamer = test_streamer().await; - streamer.config.delete_on_empty.min_age = Duration::from_secs(1); - let (msg_tx, mut msg_rx) = mpsc::unbounded_channel(); - streamer.msg_tx = msg_tx; - let (append_tx, append_rx) = oneshot::channel(); - let mut append_tx = Some(append_tx); - if !append_after_scan { - streamer.handle_append( - append_input(b"pending before scan"), - None, - append_tx.take().unwrap(), - AppendType::Regular, - ); - } - let (reply_tx, reply_rx) = oneshot::channel(); - streamer.handle_terminal_trim( - TerminalTrimCondition::DeleteOnEmpty { - expected_stream_creation_seq: 0, - expected_config_seq: 0, - }, - reply_tx, + assert_eq!( + trim_rx.await.expect("terminal trim reply").unwrap(), + TerminalTrimOutcome::Ineligible ); - let Message::DeleteOnEmptyCheckResult { - stable_pos_snapshot, - config_seq_snapshot, - records, - reply_tx, - } = msg_rx.recv().await.unwrap() - else { - panic!("expected empty-stream check result"); - }; - assert_eq!(records.as_ref().unwrap(), &RecordPresence::Empty); - if append_after_scan { - streamer.handle_append( - append_input(b"arrived during scan"), - None, - append_tx.take().unwrap(), - AppendType::Regular, - ); - } - if durable { - make_pending_append_durable(&mut streamer).await; - append_rx.await.unwrap().unwrap(); - } - // Let the minimum age elapse so it cannot mask a missing tail check. - tokio::time::sleep(Duration::from_millis(1100)).await; - streamer.handle_doe_check_result( - stable_pos_snapshot, - config_seq_snapshot, - records, - reply_tx, - ); - assert_eq!(streamer.db_writes_pending.len(), usize::from(!durable)); - assert_ne!(streamer.trim_point.state.end, SeqNum::MAX); - assert!(matches!( - reply_rx.await.unwrap().unwrap(), - TerminalTrimOutcome::RetryAt(_) - )); - streamer.db.close().await.unwrap(); - } - - #[rstest::rstest] - #[case::actor_behind(0, 1)] - #[case::worker_behind(1, 0)] - #[tokio::test] - async fn delete_on_empty_bounds_retry_for_stale_config( - #[case] actor_seq: u64, - #[case] worker_seq: u64, - ) { - let mut streamer = test_streamer().await; - streamer.config.delete_on_empty.min_age = Duration::from_secs(365 * 24 * 3600); - streamer.last_tail_write_timestamp = TimestampSecs::now(); - streamer.config_seq = actor_seq; - let earliest_retry = TimestampSecs::after(doe::RETRY_INTERVAL); - let (reply_tx, reply_rx) = oneshot::channel(); - streamer.handle_terminal_trim( - TerminalTrimCondition::DeleteOnEmpty { - expected_stream_creation_seq: 0, - expected_config_seq: worker_seq, - }, - reply_tx, - ); - let TerminalTrimOutcome::RetryAt(at) = reply_rx.await.unwrap().unwrap() else { - panic!("expected a bounded retry for mismatched configuration"); - }; - assert!(at >= earliest_retry && at <= TimestampSecs::after(doe::RETRY_INTERVAL)); - assert!(streamer.db_writes_pending.is_empty()); - streamer.db.close().await.unwrap(); - } - - #[tokio::test] - async fn delete_on_empty_bounds_retry_when_config_changes_during_scan() { - let mut streamer = test_streamer().await; - streamer.config.delete_on_empty.min_age = Duration::from_secs(365 * 24 * 3600); - streamer.last_tail_write_timestamp = TimestampSecs::now(); - let (msg_tx, mut msg_rx) = mpsc::unbounded_channel(); - streamer.msg_tx = msg_tx; - let (reply_tx, reply_rx) = oneshot::channel(); - streamer.handle_terminal_trim( - TerminalTrimCondition::DeleteOnEmpty { - expected_stream_creation_seq: 0, - expected_config_seq: 0, - }, - reply_tx, - ); - let Message::DeleteOnEmptyCheckResult { - stable_pos_snapshot, - config_seq_snapshot, - records, - reply_tx, - } = msg_rx.recv().await.unwrap() - else { - panic!("expected record check"); - }; - streamer.config_seq += 1; - let earliest_retry = TimestampSecs::after(doe::RETRY_INTERVAL); - streamer.handle_doe_check_result( - stable_pos_snapshot, - config_seq_snapshot, - records, - reply_tx, - ); - let TerminalTrimOutcome::RetryAt(at) = reply_rx.await.unwrap().unwrap() else { - panic!("expected a bounded retry for mismatched configuration"); - }; - assert!(at >= earliest_retry && at <= TimestampSecs::after(doe::RETRY_INTERVAL)); - assert!(streamer.db_writes_pending.is_empty()); - streamer.db.close().await.unwrap(); + assert_eq!(streamer.db_writes_pending.len(), 1); } #[tokio::test] @@ -2044,50 +1675,4 @@ mod tests { assert_eq!(streamer.stable_pos.seq_num, 4); assert!(streamer.inflight_appends.is_empty()); } - - #[tokio::test(start_paused = true)] - async fn cancelled_append_delays_dormancy_until_writes_are_durable() { - let mut streamer = test_streamer_with_settings(slatedb::config::Settings { - flush_interval: None, - ..Default::default() - }) - .await; - let db = streamer.db.clone(); - let (msg_tx, msg_rx) = mpsc::unbounded_channel(); - streamer.msg_tx = msg_tx; - - let (reply_tx, reply_rx) = oneshot::channel(); - streamer.handle_append( - append_input(b"cancelled"), - None, - reply_tx, - AppendType::Regular, - ); - let task = tokio::spawn(streamer.run(msg_rx)); - tokio::time::timeout(Duration::from_secs(5), async { - while db.snapshot().await.unwrap().seq() == 0 { - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - drop(reply_rx); - tokio::time::sleep(DORMANT_TIMEOUT + Duration::from_secs(1)).await; - assert_eq!( - db.status().durable_seq, - 0, - "the write must still be unflushed" - ); - assert!( - !task.is_finished(), - "dormancy abandoned an unflushed append" - ); - - db.flush().await.unwrap(); - tokio::time::timeout(DORMANT_TIMEOUT + Duration::from_secs(1), task) - .await - .expect("durable writes should allow normal dormancy") - .unwrap(); - db.close().await.unwrap(); - } } diff --git a/lite/src/backend/streams.rs b/lite/src/backend/streams.rs index b74ca6f3..b59d8032 100644 --- a/lite/src/backend/streams.rs +++ b/lite/src/backend/streams.rs @@ -1,23 +1,22 @@ use s2_common::{ basin::BasinName, - config::{DeleteOnEmptyConfig, OptionalStreamConfig, StreamConfig, StreamReconfiguration}, - record::{NonZeroSeqNum, StreamPosition}, + config::{OptionalStreamConfig, StreamConfig, StreamReconfiguration}, + record::StreamPosition, resources::{Page, ProvisionMode, ProvisionResult, RequestToken}, stream::{ListStreamsRequest, StreamInfo, StreamName}, }; use s2_storage::bash::Bash; use slatedb::{ - DbTransaction, IsolationLevel, + IsolationLevel, config::{DurabilityLevel, ScanOptions}, }; use time::OffsetDateTime; use tracing::instrument; use super::{ - Backend, doe, resolve_stream_config, - store::{db_txn_commit_durable, db_txn_get, db_txn_get_with}, - streamer::{TerminalTrimCondition, TerminalTrimOutcome}, - timestamp::TimestampSecs, + Backend, + store::db_txn_get, + streamer::{TerminalTrimCondition, TerminalTrimOutcome, doe_arm_delay}, }; use crate::{ backend::{ @@ -107,26 +106,22 @@ impl Backend { // Existence is decided from a Memory-level read; capture the row's // commit seq so exists-outcomes can await durability (see fn doc). - let (existing_meta, existing_seq) = db_txn_get_with(&txn, &stream_meta_key, |entry| { - Ok((kv::stream_meta::deser_value(entry.value)?, entry.seq)) - }) - .await? - .unzip(); - if existing_meta - .as_ref() - .is_some_and(|meta| meta.deleted_at.is_some()) - || has_terminal_trim(&txn, StreamId::new(&basin, &stream)).await? + let existing_entry = txn.get_key_value(&stream_meta_key).await?; + let existing_seq = existing_entry.as_ref().map(|kv| kv.seq); + let existing_meta = existing_entry + .map(|kv| kv::stream_meta::deser_value(kv.value)) + .transpose() + .map_err(StorageError::from)?; + if let Some(existing_meta) = &existing_meta + && existing_meta.deleted_at.is_some() { return Err(ProvisionStreamError::StreamDeletionPending( StreamDeletionPendingError, )); } - let prior_doe = existing_meta - .as_ref() - .map(|meta| meta.config.delete_on_empty); let basin_defaults = basin_meta.config.default_stream_config; - let outcome = match (existing_meta, mode) { + let (outcome, prior_doe_min_age) = match (existing_meta, mode) { (Some(existing), ProvisionMode::CreateOnly { request_token }) => { let new_creation_idempotency_key = request_token .as_ref() @@ -149,7 +144,7 @@ impl Backend { return result; } (Some(existing), ProvisionMode::Ensure) => { - let desired_config = resolve_stream_config(config, basin_defaults); + let desired_config = config.merge(basin_defaults); let config_unchanged = existing.config == desired_config; let meta = kv::stream_meta::StreamMeta { config: desired_config, @@ -158,33 +153,40 @@ impl Backend { deleted_at: None, creation_idempotency_key: existing.creation_idempotency_key, }; - if config_unchanged { - ProvisionResult::Noop(meta) - } else { - ProvisionResult::Updated(meta) - } + ( + if config_unchanged { + ProvisionResult::Noop(meta) + } else { + ProvisionResult::Updated(meta) + }, + existing.config.delete_on_empty.min_age(), + ) } (None, ProvisionMode::CreateOnly { request_token }) => { let new_creation_idempotency_key = request_token .as_ref() .map(|req_token| creation_idempotency_key(req_token, &config)); - ProvisionResult::Created(kv::stream_meta::StreamMeta { - config: resolve_stream_config(config, basin_defaults), - cipher: basin_meta.config.stream_cipher, - created_at: OffsetDateTime::now_utc(), - deleted_at: None, - creation_idempotency_key: new_creation_idempotency_key, - }) + ( + ProvisionResult::Created(kv::stream_meta::StreamMeta { + config: config.merge(basin_defaults), + cipher: basin_meta.config.stream_cipher, + created_at: OffsetDateTime::now_utc(), + deleted_at: None, + creation_idempotency_key: new_creation_idempotency_key, + }), + None, + ) } - (None, ProvisionMode::Ensure) => { + (None, ProvisionMode::Ensure) => ( ProvisionResult::Created(kv::stream_meta::StreamMeta { - config: resolve_stream_config(config, basin_defaults), + config: config.merge(basin_defaults), cipher: basin_meta.config.stream_cipher, created_at: OffsetDateTime::now_utc(), deleted_at: None, creation_idempotency_key: None, - }) - } + }), + None, + ), }; if matches!(&outcome, ProvisionResult::Noop(_)) { @@ -209,14 +211,28 @@ impl Backend { )?; } - self.commit_stream_config( - txn, - basin.clone(), - stream.clone(), - meta.config.clone(), - prior_doe, - ) - .await?; + if let Some(min_age) = meta.config.delete_on_empty.min_age() + && (matches!(&outcome, ProvisionResult::Created(_)) || prior_doe_min_age.is_none()) + { + txn.put( + kv::stream_doe_deadline::ser_key( + kv::timestamp::TimestampSecs::after(doe_arm_delay( + meta.config.retention_policy.age().unwrap_or_default(), + min_age, + )), + stream_id, + ), + kv::stream_doe_deadline::ser_value(min_age), + )?; + } + + txn.commit().await?; + } + + if let ProvisionResult::Updated(meta) = &outcome + && let Some(client) = self.streamer_client_if_active(&basin, &stream) + { + client.advise_reconfig(meta.config.clone()); } Ok(outcome.map(|meta| StreamInfo { @@ -227,6 +243,17 @@ impl Backend { })) } + pub(super) async fn stream_id_mapping( + &self, + stream_id: StreamId, + ) -> Result, StorageError> { + self.db_get( + kv::stream_id_mapping::ser_key(stream_id), + kv::stream_id_mapping::deser_value, + ) + .await + } + pub async fn get_stream_config( &self, basin: BasinName, @@ -278,60 +305,41 @@ impl Backend { stream: stream.clone(), })?; - if meta.deleted_at.is_some() - || has_terminal_trim(&txn, StreamId::new(&basin, &stream)).await? - { + if meta.deleted_at.is_some() { return Err(StreamDeletionPendingError.into()); } - let prior_doe = meta.config.delete_on_empty; + let prior_doe_min_age = meta.config.delete_on_empty.min_age(); - meta.config = resolve_stream_config( - OptionalStreamConfig::from(meta.config).reconfigure(reconfig), - basin_meta.config.default_stream_config, - ); + meta.config = OptionalStreamConfig::from(meta.config) + .reconfigure(reconfig) + .merge(basin_meta.config.default_stream_config); txn.put(&meta_key, kv::stream_meta::ser_value(&meta))?; - self.commit_stream_config(txn, basin, stream, meta.config.clone(), Some(prior_doe)) - .await?; - - Ok(meta.config) - } - - async fn commit_stream_config( - &self, - txn: DbTransaction, - basin: BasinName, - stream: StreamName, - config: StreamConfig, - prior_doe: Option, - ) -> Result<(), StorageError> { let stream_id = StreamId::new(&basin, &stream); - match (prior_doe, config.delete_on_empty.min_age()) { - (None, Some(min_age)) => { - doe::schedule(&txn, stream_id, TimestampSecs::after(min_age)).await?; - } - (Some(prior), Some(min_age)) if prior.min_age != min_age => { - doe::schedule(&txn, stream_id, TimestampSecs::now()).await?; - } - (Some(prior), None) if prior.min_age().is_some() => { - doe::clear(&txn, stream_id).await?; - } - _ => (), + if let Some(min_age) = meta.config.delete_on_empty.min_age() + && prior_doe_min_age.is_none() + { + txn.put( + kv::stream_doe_deadline::ser_key( + kv::timestamp::TimestampSecs::after(doe_arm_delay( + meta.config.retention_policy.age().unwrap_or_default(), + min_age, + )), + stream_id, + ), + kv::stream_doe_deadline::ser_value(min_age), + )?; } - let backend = self.clone(); - // Once a commit starts, cancellation must not discard its notification. - tokio::spawn(async move { - let seq = db_txn_commit_durable(txn) - .await? - .expect("stream metadata was written"); - backend.advise_stream_config(&basin, &stream, seq, config); - Ok::<_, StorageError>(()) - }) - .await - .expect("stream config commit task panicked") + txn.commit().await?; + + if let Some(client) = self.streamer_client_if_active(&basin, &stream) { + client.advise_reconfig(meta.config.clone()); + } + + Ok(meta.config) } #[instrument(ret, err, skip(self))] @@ -342,7 +350,6 @@ impl Backend { ) -> Result<(), DeleteStreamError> { self.delete_stream_with_condition(basin, stream, TerminalTrimCondition::Always) .await - .map(|_| ()) } pub(super) async fn delete_stream_with_condition( @@ -350,7 +357,7 @@ impl Backend { basin: BasinName, stream: StreamName, condition: TerminalTrimCondition, - ) -> Result { + ) -> Result<(), DeleteStreamError> { let outcome = match self.streamer_client_guarded(&basin, &stream).await { Ok(client) => client.terminal_trim(condition).await?, Err(StreamerError::Storage(e)) => { @@ -361,10 +368,10 @@ impl Backend { } Err(StreamerError::StreamDeletionPending(_)) => TerminalTrimOutcome::DeletionPending, }; - if outcome == TerminalTrimOutcome::DeletionPending { - self.mark_stream_deleted(basin, stream).await?; + match outcome { + TerminalTrimOutcome::DeletionPending => self.mark_stream_deleted(basin, stream).await, + TerminalTrimOutcome::Ineligible => Ok(()), } - Ok(outcome) } async fn mark_stream_deleted( @@ -373,49 +380,22 @@ impl Backend { stream: StreamName, ) -> Result<(), DeleteStreamError> { let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; - // A delayed deletion request may resume after the trim worker has deleted the old - // stream and the name has been reused. Only mark metadata when this - // transaction also sees a terminal trim marker. - if !has_terminal_trim(&txn, StreamId::new(&basin, &stream)).await? { - let read_seq = txn.seqnum(); - drop(txn); - // The trim worker may have removed the marker without flushing yet. - // Wait for that removal to become durable before acknowledging deletion. - self.await_durable_seq(read_seq).await?; - return Ok(()); - } let meta_key = kv::stream_meta::ser_key(&basin, &stream); - let (mut meta, seq) = db_txn_get_with(&txn, &meta_key, |entry| { - Ok((kv::stream_meta::deser_value(entry.value)?, entry.seq)) - }) - .await? - .ok_or_else(|| StreamNotFoundError { - basin, - stream: stream.clone(), - })?; + let mut meta = db_txn_get(&txn, &meta_key, kv::stream_meta::deser_value) + .await? + .ok_or_else(|| StreamNotFoundError { + basin, + stream: stream.clone(), + })?; if meta.deleted_at.is_none() { meta.deleted_at = Some(OffsetDateTime::now_utc()); txn.put(&meta_key, kv::stream_meta::ser_value(&meta))?; - db_txn_commit_durable(txn).await?; - } else { - // The terminal trim is durable, but the metadata marker may not be yet. - drop(txn); - self.await_durable_seq(seq).await?; + txn.commit().await?; } Ok(()) } } -async fn has_terminal_trim(txn: &DbTransaction, stream_id: StreamId) -> Result { - Ok(db_txn_get( - txn, - kv::stream_trim_point::ser_key(stream_id), - kv::stream_trim_point::deser_value, - ) - .await? - == Some(..NonZeroSeqNum::MAX)) -} - fn creation_idempotency_key(req_token: &RequestToken, config: &OptionalStreamConfig) -> Bash { Bash::length_prefixed(&[ req_token.as_bytes(), diff --git a/lite/src/backend/test_util.rs b/lite/src/backend/test_util.rs deleted file mode 100644 index 3c45f38a..00000000 --- a/lite/src/backend/test_util.rs +++ /dev/null @@ -1,44 +0,0 @@ -use std::future::Future; - -use slatedb::{Error, WriteHandle}; - -/// Finish a fixture write and assert that remote reads can observe it. -pub(super) trait DbWriteTestExt: - Future> + Sized -{ - async fn assert_durable(self) { - self.await - .expect("fixture write should succeed") - .await_durable() - .await - .expect("fixture write should become durable"); - } -} - -impl>> DbWriteTestExt for F {} - -/// Provision a real stream for lifecycle tests, including its ID mapping and tail. -pub(super) async fn create_stream( - backend: &super::Backend, - config: s2_common::config::OptionalStreamConfig, -) -> (s2_common::basin::BasinName, s2_common::stream::StreamName) { - use s2_common::{config::BasinConfig, resources::ProvisionMode}; - let basin: s2_common::basin::BasinName = "test-basin".parse().unwrap(); - let stream: s2_common::stream::StreamName = "test-stream".parse().unwrap(); - backend - .provision_basin(basin.clone(), BasinConfig::default(), ProvisionMode::Ensure) - .await - .unwrap(); - backend - .provision_stream( - basin.clone(), - stream.clone(), - config, - ProvisionMode::CreateOnly { - request_token: None, - }, - ) - .await - .unwrap(); - (basin, stream) -} diff --git a/lite/src/handlers/v1/error.rs b/lite/src/handlers/v1/error.rs index e589edc2..b25e34b0 100644 --- a/lite/src/handlers/v1/error.rs +++ b/lite/src/handlers/v1/error.rs @@ -177,7 +177,7 @@ impl ServiceError { } DeleteStreamError::RequestDroppedError(e) => { // Unavailable error code promised to be side-effect free, - // The terminal trim marker may have become durable prior to drop. + // AppendType::Terminal may have become durable prior to drop. standard(ErrorCode::Other, e.to_string()) } DeleteStreamError::StreamNotFound(e) => { @@ -239,7 +239,7 @@ impl ServiceError { } AppendError::RequestDroppedError(e) => { // Unavailable error code promised to be side-effect free, - // The append may have become durable prior to drop. + // AppendType::Regular may have become durable prior to drop. standard(ErrorCode::Other, e.to_string()) } AppendError::BasinNotFound(e) => standard(ErrorCode::BasinNotFound, e.to_string()), diff --git a/lite/src/handlers/v1/records.rs b/lite/src/handlers/v1/records.rs index 10fe2907..5532e20f 100644 --- a/lite/src/handlers/v1/records.rs +++ b/lite/src/handlers/v1/records.rs @@ -46,15 +46,11 @@ fn validate_read_until(start: ReadStart, end: ReadEnd) -> Result<(), ServiceErro Ok(()) } -/// Adjusts the read bounds to resume after `last_event_id`, and returns the -/// count/bytes already delivered so emitted event ids stay cumulative across -/// reconnects. fn apply_last_event_id( mut start: ReadStart, mut end: v1t::stream::ReadEnd, last_event_id: Option, -) -> (ReadStart, v1t::stream::ReadEnd, CountOrBytes) { - let mut delivered = CountOrBytes::ZERO; +) -> (ReadStart, v1t::stream::ReadEnd) { if let Some(v1t::stream::sse::LastEventId { seq_num, count, @@ -64,9 +60,8 @@ fn apply_last_event_id( start.from = ReadFrom::SeqNum(seq_num.saturating_add(1)); end.count = end.count.map(|c| c.saturating_sub(count)); end.bytes = end.bytes.map(|c| c.saturating_sub(bytes)); - delivered = CountOrBytes { count, bytes }; } - (start, end, delivered) + (start, end) } enum ReadMode { @@ -166,7 +161,6 @@ pub struct ReadArgs { v1t::StreamNamePathSegment, s2_api::data::S2FormatHeader, s2_api::data::S2EncryptionKeyHeader, - s2_api::data::S2StreamConfigHeader, v1t::stream::ReadStart, v1t::stream::ReadEnd, ), @@ -192,13 +186,12 @@ pub async fn read( match request { v1t::stream::ReadRequest::Unary { encryption_key, - create_stream_config_patch, format, response_mime, } => { let (start, end) = prepare_read(start, end, ReadMode::Unary)?; let session = backend - .open_for_read(&basin, &stream, encryption_key, create_stream_config_patch) + .open_for_read(&basin, &stream, encryption_key) .await? .read(start, end) .await?; @@ -216,19 +209,18 @@ pub async fn read( } v1t::stream::ReadRequest::EventStream { encryption_key, - create_stream_config_patch, format, last_event_id, } => { - let (start, end, delivered) = apply_last_event_id(start, end, last_event_id); + let (start, end) = apply_last_event_id(start, end, last_event_id); let (start, end) = prepare_read(start, end, ReadMode::Streaming)?; let session = backend - .open_for_read(&basin, &stream, encryption_key, create_stream_config_patch) + .open_for_read(&basin, &stream, encryption_key) .await? .read(start, end) .await?; let events = async_stream::stream! { - let mut processed = delivered; + let mut processed = CountOrBytes::ZERO; tokio::pin!(session); let mut errored = false; while let Some(output) = session.next().await { @@ -240,8 +232,8 @@ pub async fn read( let Some(last_record) = batch.records.last() else { continue; }; - processed.count = processed.count.saturating_add(batch.records.len()); - processed.bytes = processed.bytes.saturating_add(batch.records.metered_size()); + processed.count += batch.records.len(); + processed.bytes += batch.records.metered_size(); let id = v1t::stream::sse::LastEventId { seq_num: last_record.position().seq_num, count: processed.count, @@ -273,12 +265,11 @@ pub async fn read( } v1t::stream::ReadRequest::S2s { encryption_key, - create_stream_config_patch, response_compression, } => { let (start, end) = prepare_read(start, end, ReadMode::Streaming)?; let s2s_stream = backend - .open_for_read(&basin, &stream, encryption_key, create_stream_config_patch) + .open_for_read(&basin, &stream, encryption_key) .await? .read(start, end) .await? @@ -373,7 +364,6 @@ pub struct AppendArgs { v1t::StreamNamePathSegment, s2_api::data::S2FormatHeader, s2_api::data::S2EncryptionKeyHeader, - s2_api::data::S2StreamConfigHeader, ), servers( (url = super::paths::cloud_endpoints::BASIN, variables( @@ -394,12 +384,11 @@ pub async fn append( match request { v1t::stream::AppendRequest::Unary { encryption_key, - create_stream_config_patch, input, response_mime, } => { let handle = backend - .open_for_append(&basin, &stream, encryption_key, create_stream_config_patch) + .open_for_append(&basin, &stream, encryption_key) .await?; let ack = handle.append(input).await?; match response_mime { @@ -415,12 +404,11 @@ pub async fn append( } v1t::stream::AppendRequest::S2s { encryption_key, - create_stream_config_patch, inputs, response_compression, } => { let handle = backend - .open_for_append(&basin, &stream, encryption_key, create_stream_config_patch) + .open_for_append(&basin, &stream, encryption_key) .await?; let (err_tx, err_rx) = tokio::sync::oneshot::channel(); @@ -481,22 +469,16 @@ mod tests { use bytesize::ByteSize; use futures::TryStreamExt as _; use prost::Message as _; - use s2_api::v1::{ - config::STREAM_CONFIG_HEADER, - stream::{ - proto, - s2s::{self, FrameDecoder, SessionMessage}, - sse::LastEventId, - }, + use s2_api::v1::stream::{ + proto, + s2s::{FrameDecoder, SessionMessage}, }; use s2_common::{ basin::{BASIN_HEADER, BasinName}, - config::{ - BasinConfig, DeleteOnEmptyConfig, OptionalStreamConfig, RetentionPolicy, StreamConfig, - }, + config::{BasinConfig, OptionalStreamConfig}, encryption::{EncryptionAlgorithm, EncryptionKey, S2_ENCRYPTION_KEY_HEADER}, read_extent::{ReadLimit, ReadUntil}, - record::{EnvelopeRecord, Metered, MeteredSize as _, Record}, + record::{EnvelopeRecord, Metered, Record}, resources::ProvisionMode, stream::{ AppendInput, AppendRecord, AppendRecordBatch, AppendRecordParts, ListStreamsRequest, @@ -615,12 +597,7 @@ mod tests { encryption_key: EncryptionKey, ) { backend - .open_for_append( - basin, - stream, - Some(encryption_key), - OptionalStreamConfig::default(), - ) + .open_for_append(basin, stream, Some(encryption_key)) .await .expect("open append handle") .append(append_input(body)) @@ -628,95 +605,6 @@ mod tests { .expect("append encrypted payload"); } - async fn append_payload( - backend: &Backend, - basin: &BasinName, - stream: &StreamName, - body: &'static [u8], - ) { - backend - .open_for_append(basin, stream, None, OptionalStreamConfig::default()) - .await - .expect("open append handle") - .append(append_input(body)) - .await - .expect("append payload"); - } - - struct SseBatches { - seq_nums: Vec, - last_id: Option, - } - - async fn sse_read( - app: &axum::Router, - basin: &BasinName, - stream: &StreamName, - bounds: &str, - last_event_id: Option, - ) -> SseBatches { - let mut request = request_builder( - "GET", - format!("/v1/streams/{stream}/records?seq_num=0&wait=0&{bounds}"), - basin, - ) - .header(header::ACCEPT, "text/event-stream"); - if let Some(id) = last_event_id { - request = request.header("last-event-id", id.to_string()); - } - let response = send(app, request.body(Body::empty()).unwrap()).await; - assert_eq!(response.status(), StatusCode::OK); - let body = - tokio::time::timeout(Duration::from_secs(5), response_bytes(response, "sse body")) - .await - .expect("SSE read should terminate"); - let body = std::str::from_utf8(&body).expect("utf8 sse body"); - - let mut batches = SseBatches { - seq_nums: Vec::new(), - last_id: None, - }; - let mut done = false; - for event in body.split("\n\n").filter(|e| !e.trim().is_empty()) { - let field = |name: &str| { - event - .lines() - .find_map(|line| line.strip_prefix(name)) - .map(str::trim) - }; - match field("event:") { - Some("batch") => { - let id: LastEventId = field("id:") - .expect("batch event id") - .parse() - .expect("parse batch event id"); - let data: serde_json::Value = - serde_json::from_str(field("data:").expect("batch event data")) - .expect("batch event json"); - let records = data["records"].as_array().expect("records array"); - batches.seq_nums.extend( - records - .iter() - .map(|r| r["seq_num"].as_u64().expect("seq_num")), - ); - assert_eq!( - id.seq_num, - *batches.seq_nums.last().expect("non-empty batch") - ); - batches.last_id = Some(id); - } - Some("error") => panic!("unexpected sse error event: {event}"), - Some(other) => panic!("unexpected sse event `{other}`: {event}"), - None => { - assert_eq!(field("data:"), Some("[DONE]"), "unexpected event: {event}"); - done = true; - } - } - } - assert!(done, "SSE read should end with [DONE]"); - batches - } - fn read_uri(stream: &StreamName) -> String { format!("/v1/streams/{stream}/records?seq_num=0&wait=0") } @@ -809,12 +697,7 @@ mod tests { assert_eq!(ack.end.as_ref().map(|pos| pos.seq_num), Some(1)); let records = backend - .open_for_read( - &basin, - &stream, - Some(encryption_key.clone()), - OptionalStreamConfig::default(), - ) + .open_for_read(&basin, &stream, Some(encryption_key.clone())) .await .expect("open read handle") .read( @@ -848,240 +731,6 @@ mod tests { assert_eq!(record.body().as_ref(), b"secret"); } - fn basin_config_with_create_stream_on_append() -> BasinConfig { - BasinConfig { - create_stream_on_append: true, - default_stream_config: OptionalStreamConfig { - storage_class: Some("standard".into()), - ..Default::default() - }, - ..Default::default() - } - } - - fn expected_auto_created_config() -> StreamConfig { - StreamConfig { - storage_class: Some("standard".into()), - retention_policy: RetentionPolicy::Age(Duration::from_secs(3600)), - timestamping: Default::default(), - delete_on_empty: DeleteOnEmptyConfig { - min_age: Duration::from_secs(300), - }, - } - } - - const STREAM_CONFIG_HEADER_VALUE: &str = - r#"{"retention_policy":{"age":3600},"delete_on_empty":{"min_age_secs":300}}"#; - - fn append_record_input(body: &'static [u8]) -> proto::AppendInput { - proto::AppendInput { - records: vec![proto::AppendRecord { - timestamp: None, - headers: vec![], - body: Bytes::from_static(body), - }], - match_seq_num: None, - fencing_token: None, - } - } - - #[tokio::test] - async fn json_append_auto_creates_stream_with_stream_config_header() { - let (app, backend, basin, stream) = setup_app_without_stream( - "append-json-create-config", - basin_config_with_create_stream_on_append(), - ) - .await; - - let body = serde_json::json!({"records": [{"body": "hello"}]}); - let response = send( - &app, - request_builder("POST", format!("/v1/streams/{stream}/records"), &basin) - .header(header::CONTENT_TYPE, "application/json") - .header(STREAM_CONFIG_HEADER.as_str(), STREAM_CONFIG_HEADER_VALUE) - .body(Body::from(body.to_string())) - .unwrap(), - ) - .await; - - assert_eq!(response.status(), StatusCode::OK); - let ack = response_json(response, "append ack body").await; - assert_eq!(ack["end"]["seq_num"], 1); - let config = backend - .get_stream_config(basin, stream) - .await - .expect("get stream config"); - assert_eq!(config, expected_auto_created_config()); - } - - #[tokio::test] - async fn append_with_invalid_stream_config_header_is_rejected_without_creating() { - let (app, backend, basin, stream) = setup_app_without_stream( - "append-create-config-invalid", - basin_config_with_create_stream_on_append(), - ) - .await; - - for (value, expected_message) in [ - ( - r#"{"retention_policy":{"age":0}}"#, - "age must be greater than 0 seconds", - ), - ("not json", "invalid JSON"), - ] { - let body = serde_json::json!({"records": [{"body": "hello"}]}); - let response = send( - &app, - request_builder("POST", format!("/v1/streams/{stream}/records"), &basin) - .header(header::CONTENT_TYPE, "application/json") - .header(STREAM_CONFIG_HEADER.as_str(), value) - .body(Body::from(body.to_string())) - .unwrap(), - ) - .await; - - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - let info = response_json(response, "append error body").await; - assert_eq!(info["code"], "bad_header"); - let message = info["message"].as_str().expect("error message string"); - assert!( - message.contains("s2-stream-config") && message.contains(expected_message), - "{message}" - ); - } - assert_no_streams(&backend, &basin).await; - } - - #[tokio::test] - async fn s2s_append_session_auto_creates_stream_with_stream_config_header() { - let (app, backend, basin, stream) = setup_app_without_stream( - "append-s2s-create-config", - basin_config_with_create_stream_on_append(), - ) - .await; - - let frame = |body: &'static [u8]| { - SessionMessage::regular(s2s::CompressionAlgorithm::None, &append_record_input(body)) - .expect("encode frame") - .encode() - }; - let mut body = BytesMut::new(); - body.extend_from_slice(&frame(b"first")); - body.extend_from_slice(&frame(b"second")); - - let response = send( - &app, - request_builder("POST", format!("/v1/streams/{stream}/records"), &basin) - .header(header::CONTENT_TYPE, "s2s/proto") - .header(STREAM_CONFIG_HEADER.as_str(), STREAM_CONFIG_HEADER_VALUE) - .body(Body::from(body.freeze())) - .unwrap(), - ) - .await; - - assert_eq!(response.status(), StatusCode::OK); - let body = response_bytes(response, "s2s body").await; - let mut decoder = FrameDecoder; - let mut buf = BytesMut::from(body.as_ref()); - let mut acks = Vec::new(); - while let Some(frame) = decoder.decode(&mut buf).expect("frame decode") { - let SessionMessage::Regular(ack) = frame else { - panic!("expected regular frame"); - }; - acks.push( - ack.try_into_proto::() - .expect("decode append ack"), - ); - } - assert_eq!(acks.len(), 2); - assert_eq!(acks[1].end.as_ref().map(|pos| pos.seq_num), Some(2)); - - let config = backend - .get_stream_config(basin, stream) - .await - .expect("get stream config"); - assert_eq!(config, expected_auto_created_config()); - } - - #[rstest::rstest] - #[case::count("count=0")] - #[case::bytes("bytes=0")] - #[tokio::test] - async fn unary_read_with_zero_limit_returns_empty( - #[case] bound: &str, - #[values( - "timestamp=0", - "seq_num=0", - "tail_offset=0", - "timestamp=1", - "seq_num=1" - )] - start: &str, - #[values(0, 60)] wait: u32, - ) { - let (app, _backend, basin, stream) = setup_app_with_config( - "read-zero-limit", - BasinConfig::default(), - OptionalStreamConfig::default(), - ) - .await; - - let response = tokio::time::timeout( - Duration::from_secs(2), - send( - &app, - request_builder( - "GET", - format!("/v1/streams/{stream}/records?{start}&{bound}&wait={wait}"), - &basin, - ) - .body(Body::empty()) - .unwrap(), - ), - ) - .await - .expect("a zero limit should complete without waiting for records"); - - let status = response.status(); - let body = response_json(response, "read zero-limit response").await; - assert_eq!( - status, - StatusCode::OK, - "{start}&{bound}&wait={wait}: {body}" - ); - assert_eq!(body["records"], serde_json::json!([])); - } - - #[tokio::test] - async fn read_auto_creates_stream_with_stream_config_header() { - let basin_config = BasinConfig { - create_stream_on_append: false, - create_stream_on_read: true, - ..basin_config_with_create_stream_on_append() - }; - let (app, backend, basin, stream) = - setup_app_without_stream("read-create-config", basin_config).await; - - let response = send( - &app, - request_builder( - "GET", - format!("/v1/streams/{stream}/records?seq_num=0"), - &basin, - ) - .header(STREAM_CONFIG_HEADER.as_str(), STREAM_CONFIG_HEADER_VALUE) - .body(Body::empty()) - .unwrap(), - ) - .await; - assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); - let config = backend - .get_stream_config(basin, stream) - .await - .expect("get stream config"); - assert_eq!(config, expected_auto_created_config()); - } - #[tokio::test] async fn invalid_read_bounds_do_not_auto_create_stream() { let basin_config = BasinConfig { @@ -1185,103 +834,6 @@ mod tests { ); } - #[rstest::rstest] - #[case::count(Some(10), None)] - #[case::bytes(None, Some(10))] - #[case::count_first(Some(10), Some(15))] - #[case::bytes_first(Some(15), Some(10))] - #[tokio::test] - async fn sse_resume_emits_cumulative_ids_and_honors_bounds( - #[case] count: Option, - #[case] bytes_in_records: Option, - ) { - let (app, backend, basin, stream) = setup_app_with_config( - "read-sse-resume", - BasinConfig::default(), - OptionalStreamConfig::default(), - ) - .await; - let per_record = append_input(b"payload").records.metered_size(); - let mut bounds = String::new(); - if let Some(count) = count { - bounds.push_str(&format!("count={count}&")); - } - if let Some(records) = bytes_in_records { - bounds.push_str(&format!("bytes={}&", records * per_record)); - } - - // End two connections at the current tail, then resume with exactly - // the same bounds and the actual id emitted by the previous connection. - let mut last_id = None; - let mut delivered = Vec::new(); - for (appended, expected_total) in [(4, 4), (3, 7), (13, 10)] { - for _ in 0..appended { - append_payload(&backend, &basin, &stream, b"payload").await; - } - let resumed = sse_read(&app, &basin, &stream, &bounds, last_id).await; - delivered.extend(resumed.seq_nums); - assert_eq!(delivered, (0..expected_total as u64).collect::>()); - let id = resumed.last_id.expect("last id"); - assert_eq!(id.seq_num, expected_total as u64 - 1); - assert_eq!(id.count, expected_total); - assert_eq!(id.bytes, expected_total * per_record); - last_id = Some(id); - } - - // The same read without reconnects produces the same records and id. - let full = sse_read(&app, &basin, &stream, &bounds, None).await; - assert_eq!(full.seq_nums, delivered); - assert_eq!( - full.last_id.expect("full id").to_string(), - last_id.expect("resumed id").to_string(), - ); - - let exhausted = sse_read(&app, &basin, &stream, &bounds, last_id).await; - assert!(exhausted.seq_nums.is_empty(), "{:?}", exhausted.seq_nums); - assert!(exhausted.last_id.is_none()); - } - - #[rstest::rstest] - #[case::unbounded("", (usize::MAX - 1, usize::MAX - 1), (usize::MAX, usize::MAX), 3)] - #[case::count_bound("count=2", (0, usize::MAX), (2, usize::MAX), 2)] - #[case::bytes_bound("bytes=30", (usize::MAX, 0), (usize::MAX, 30), 2)] - #[tokio::test] - async fn sse_resume_saturates_client_supplied_counters( - #[case] bounds: &str, - #[case] (count, bytes): (usize, usize), - #[case] expected_totals: (usize, usize), - #[case] expected_records: usize, - ) { - let (app, backend, basin, stream) = setup_app_with_config( - "read-sse-saturation", - BasinConfig::default(), - OptionalStreamConfig::default(), - ) - .await; - // Each "payload" record occupies 15 metered bytes. - for _ in 0..4 { - append_payload(&backend, &basin, &stream, b"payload").await; - } - let resumed = sse_read( - &app, - &basin, - &stream, - bounds, - Some(LastEventId { - seq_num: 0, - count, - bytes, - }), - ) - .await; - assert_eq!( - resumed.seq_nums, - (1..=expected_records as u64).collect::>() - ); - let id = resumed.last_id.expect("last id"); - assert_eq!((id.count, id.bytes), expected_totals); - } - #[tokio::test] async fn s2s_read_without_key_header_is_rejected_before_stream_starts() { let encryption_key = aegis_key(0x42); diff --git a/lite/src/handlers/v1/streams.rs b/lite/src/handlers/v1/streams.rs index 1792da12..4f694284 100644 --- a/lite/src/handlers/v1/streams.rs +++ b/lite/src/handlers/v1/streams.rs @@ -338,157 +338,3 @@ pub async fn reconfigure_stream( .await?; Ok(Json(config.into())) } - -#[cfg(test)] -mod test { - use std::{sync::Arc, time::Duration}; - - use axum::{ - body::{self, Body}, - http::{Request, StatusCode, header}, - response::Response, - }; - use bytesize::ByteSize; - use rstest::rstest; - use s2_common::{ - basin::{BASIN_HEADER, BasinName}, - config::BasinConfig, - resources::ProvisionMode, - }; - use slatedb::{Db, config::Settings, object_store::memory::InMemory}; - use tower::ServiceExt as _; - use uuid::Uuid; - - use crate::{backend::Backend, handlers}; - - async fn setup_app(test_suffix: &str) -> (axum::Router, Backend, BasinName) { - let object_store = Arc::new(InMemory::new()); - let db_path = format!("/tmp/streams-handler-test-{}", Uuid::new_v4()); - let db = Db::builder(db_path, object_store) - .with_settings(Settings { - flush_interval: Some(Duration::from_millis(5)), - ..Default::default() - }) - .build() - .await - .expect("create in-memory db"); - let backend = Backend::new(db, ByteSize::mib(10)); - let basin: BasinName = format!("test-basin-{test_suffix}").parse().unwrap(); - backend - .provision_basin( - basin.clone(), - BasinConfig::default(), - ProvisionMode::CreateOnly { - request_token: None, - }, - ) - .await - .expect("create basin"); - let app = handlers::router().with_state(backend.clone()); - (app, backend, basin) - } - - async fn send(app: &axum::Router, request: Request) -> Response { - app.clone() - .oneshot(request) - .await - .expect("request should complete") - } - - async fn response_json(response: Response) -> serde_json::Value { - let body = body::to_bytes(response.into_body(), usize::MAX) - .await - .expect("response body"); - serde_json::from_slice(&body).expect("json body") - } - - async fn create_stream(app: &axum::Router, basin: &BasinName, name: &str) -> Response { - let payload = serde_json::json!({ "stream": name }).to_string(); - send( - app, - Request::builder() - .method("POST") - .uri("/v1/streams") - .header(BASIN_HEADER.as_str(), basin.as_ref()) - .header(header::CONTENT_TYPE, "application/json") - .body(Body::from(payload)) - .unwrap(), - ) - .await - } - - async fn list_streams(app: &axum::Router, basin: &BasinName, query: &str) -> Response { - send( - app, - Request::builder() - .method("GET") - .uri(format!("/v1/streams?{query}")) - .header(BASIN_HEADER.as_str(), basin.as_ref()) - .body(Body::empty()) - .unwrap(), - ) - .await - } - - #[tokio::test] - async fn create_stream_with_nul_byte_is_bad_json() { - let (app, backend, basin) = setup_app("create-nul").await; - - let response = create_stream(&app, &basin, "a\0b").await; - - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - let error = response_json(response).await; - assert_eq!(error["code"], "bad_json"); - assert!( - error["message"] - .as_str() - .unwrap() - .contains("stream name must not contain NUL bytes"), - "unexpected message: {error}" - ); - - backend.close().await.expect("close backend"); - } - - #[rstest] - #[case::control_chars("a\tb\nc\rd\x01e")] - #[case::unicode("stream/名前 😀?#%20")] - #[tokio::test] - async fn create_stream_with_other_chars_is_created(#[case] name: &str) { - let (app, backend, basin) = setup_app("create-ok").await; - - let response = create_stream(&app, &basin, name).await; - - assert_eq!(response.status(), StatusCode::CREATED); - let info = response_json(response).await; - assert_eq!(info["name"], name); - - backend.close().await.expect("close backend"); - } - - #[rstest] - #[case::prefix("prefix=a%00b", "stream prefix must not contain NUL bytes")] - #[case::start_after("start_after=a%00b", "stream start-after must not contain NUL bytes")] - #[tokio::test] - async fn list_streams_with_nul_byte_is_bad_query( - #[case] query: &str, - #[case] expected_message: &str, - ) { - let (app, backend, basin) = setup_app("list-nul").await; - - let response = list_streams(&app, &basin, query).await; - - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - let error = response_json(response).await; - assert_eq!(error["code"], "bad_query"); - assert!( - error["message"] - .as_str() - .unwrap() - .contains(expected_message), - "unexpected message: {error}" - ); - - backend.close().await.expect("close backend"); - } -} diff --git a/lite/src/server.rs b/lite/src/server.rs index d78f71f7..7594684e 100644 --- a/lite/src/server.rs +++ b/lite/src/server.rs @@ -42,47 +42,15 @@ pub struct LiteArgs { /// Name of the S3 bucket to back the database. /// /// If not specified, in-memory storage is used unless --local-root is set. - /// Uses the standard AWS configuration for the endpoint, region and credentials. - #[arg(long, group = "main_store")] + #[arg(long)] pub bucket: Option, /// Root directory to back the database on the local filesystem. /// /// Conflicts with --bucket. - #[arg( - long, - value_name = "DIR", - conflicts_with = "bucket", - group = "main_store" - )] + #[arg(long, value_name = "DIR", conflicts_with = "bucket")] pub local_root: Option, - /// Name of the S3 bucket to back the write-ahead log (WAL). - /// - /// If not specified, the main store is used unless --wal-local-root is set. - /// Uses the same AWS configuration as --bucket, with optional - /// S2LITE_WAL_AWS_* overrides for the endpoint, region and credentials. - /// - /// Requires --bucket or --local-root. Conflicts with --wal-local-root. - #[arg( - long, - env = "S2LITE_WAL_BUCKET", - requires = "main_store", - conflicts_with = "wal_local_root" - )] - pub wal_bucket: Option, - - /// Root directory to back the write-ahead log (WAL) on the local filesystem. - /// - /// Requires --bucket or --local-root. Conflicts with --wal-bucket. - #[arg( - long, - env = "S2LITE_WAL_LOCAL_ROOT", - value_name = "DIR", - requires = "main_store" - )] - pub wal_local_root: Option, - /// Base path on object storage. #[arg(long, default_value = "")] pub path: String, @@ -205,44 +173,22 @@ pub async fn run(args: LiteArgs) -> eyre::Result<()> { StoreType::InMemory }; - let object_store = init_object_store(&store_type, None).await?; - let wal_store_type = if let Some(bucket) = args.wal_bucket { - Some(StoreType::S3Bucket(bucket)) - } else { - args.wal_local_root.map(StoreType::LocalFileSystem) - }; - let wal_object_store = match &wal_store_type { - Some(wal_store_type) => { - let s3_overrides = match wal_store_type { - StoreType::S3Bucket(_) => Some(WalS3Overrides::from_env()?), - StoreType::LocalFileSystem(_) | StoreType::InMemory => None, - }; - Some(init_object_store(wal_store_type, s3_overrides).await?) - } - None => None, - }; + let object_store = init_object_store(&store_type).await?; let db_settings = slatedb::Settings::from_env_with_default( "SL8_", slatedb::Settings { - flush_interval: Some( - wal_store_type - .as_ref() - .unwrap_or(&store_type) - .default_flush_interval(), - ), + flush_interval: Some(store_type.default_flush_interval()), ..Default::default() }, )?; let manifest_poll_interval = db_settings.manifest_poll_interval; - let mut builder = slatedb::Db::builder(args.path, object_store).with_settings(db_settings); - if let Some(wal_object_store) = wal_object_store { - info!(store = ?wal_store_type, "using dedicated WAL object store"); - builder = builder.with_wal_object_store(wal_object_store); - } - let db = builder.build().await?; + let db = slatedb::Db::builder(args.path, object_store) + .with_settings(db_settings) + .build() + .await?; info!( ?manifest_poll_interval, @@ -253,7 +199,6 @@ pub async fn run(args: LiteArgs) -> eyre::Result<()> { info!(%args.append_inflight_bytes, "starting backend"); let backend = Backend::new(db, args.append_inflight_bytes); - let shutdown_backend = backend.clone(); crate::backend::bgtasks::spawn(&backend); if let Some(init_file) = &args.init_file { @@ -333,32 +278,63 @@ pub async fn run(args: LiteArgs) -> eyre::Result<()> { } } - info!("http server stopped; closing SlateDB"); - let close_started = Instant::now(); - shutdown_backend - .close() - .await - .map_err(|error| eyre::eyre!("SlateDB close: {error}"))?; - info!( - elapsed_ms = close_started.elapsed().as_millis(), - "SlateDB closed" - ); - Ok(()) } async fn init_object_store( store_type: &StoreType, - s3_overrides: Option, ) -> eyre::Result> { - let store: Arc = match store_type { + Ok(match store_type { StoreType::S3Bucket(bucket) => { info!(bucket, "using s3 object store"); - let mut builder = s3_builder().await; - if let Some(overrides) = s3_overrides { - builder = overrides.apply(builder); + let mut builder = + object_store::aws::AmazonS3Builder::from_env().with_bucket_name(bucket); + + if let Some(endpoint) = + std::env::var_os("AWS_ENDPOINT_URL_S3").and_then(|s| s.into_string().ok()) + { + if endpoint.starts_with("http://") { + builder = builder.with_allow_http(true); + } + builder = builder.with_endpoint(endpoint); } - Arc::new(builder.with_bucket_name(bucket).build()?) + + match ( + std::env::var_os("AWS_ACCESS_KEY_ID").and_then(|s| s.into_string().ok()), + std::env::var_os("AWS_SECRET_ACCESS_KEY").and_then(|s| s.into_string().ok()), + ) { + (Some(key_id), Some(secret_key)) => { + info!(key_id, "using static credentials from env vars"); + + let token = + std::env::var_os("AWS_SESSION_TOKEN").and_then(|s| s.into_string().ok()); + builder = builder.with_credentials(Arc::new( + object_store::StaticCredentialProvider::new( + object_store::aws::AwsCredential { + key_id, + secret_key, + token, + }, + ), + )); + } + _ => { + let aws_config = + aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; + if let Some(region) = aws_config.region() { + info!(region = region.as_ref()); + builder = builder.with_region(region.to_string()); + } + if let Some(credentials_provider) = aws_config.credentials_provider() { + info!("using aws-config credentials provider"); + builder = builder.with_credentials(Arc::new(S3CredentialProvider { + aws: credentials_provider.clone(), + cache: tokio::sync::Mutex::new(None), + })); + } + } + } + Arc::new(builder.build()?) as Arc } StoreType::LocalFileSystem(local_root) => { std::fs::create_dir_all(local_root)?; @@ -376,127 +352,7 @@ async fn init_object_store( info!("using in-memory object store"); Arc::new(object_store::memory::InMemory::new()) } - }; - Ok(store) -} - -// Both buckets start with the same AWS configuration and credential chain. -async fn s3_builder() -> object_store::aws::AmazonS3Builder { - let mut builder = object_store::aws::AmazonS3Builder::from_env(); - - if let Some(endpoint) = - std::env::var_os("AWS_ENDPOINT_URL_S3").and_then(|s| s.into_string().ok()) - { - if endpoint.starts_with("http://") { - builder = builder.with_allow_http(true); - } - builder = builder.with_endpoint(endpoint); - } - - match ( - std::env::var_os("AWS_ACCESS_KEY_ID").and_then(|s| s.into_string().ok()), - std::env::var_os("AWS_SECRET_ACCESS_KEY").and_then(|s| s.into_string().ok()), - ) { - (Some(key_id), Some(secret_key)) => { - info!(key_id, "using static credentials from env vars"); - - let aws_config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; - if let Some(region) = aws_config.region() { - info!(region = region.as_ref()); - builder = builder.with_region(region.to_string()); - } - - let token = std::env::var_os("AWS_SESSION_TOKEN").and_then(|s| s.into_string().ok()); - builder = builder.with_credentials(Arc::new( - object_store::StaticCredentialProvider::new(object_store::aws::AwsCredential { - key_id, - secret_key, - token, - }), - )); - } - _ => { - let aws_config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; - if let Some(region) = aws_config.region() { - info!(region = region.as_ref()); - builder = builder.with_region(region.to_string()); - } - if let Some(credentials_provider) = aws_config.credentials_provider() { - info!("using aws-config credentials provider"); - builder = builder.with_credentials(Arc::new(S3CredentialProvider { - aws: credentials_provider.clone(), - cache: tokio::sync::Mutex::new(None), - })); - } - } - } - builder -} - -// Keep credentials out of LiteArgs and its startup Debug log. Only supplied -// fields override the shared AWS configuration; credentials are replaced as a set. -struct WalS3Overrides { - endpoint: Option, - region: Option, - access_key_id: Option, - secret_access_key: Option, - session_token: Option, -} - -impl WalS3Overrides { - fn from_env() -> eyre::Result { - Self::from_getter(|name| match std::env::var(name) { - Ok(value) => Ok(Some(value)), - Err(std::env::VarError::NotPresent) => Ok(None), - Err(std::env::VarError::NotUnicode(_)) => { - Err(eyre::eyre!("{name} must contain valid UTF-8")) - } - }) - } - - fn from_getter(get: impl Fn(&str) -> eyre::Result>) -> eyre::Result { - let config = Self { - endpoint: get("S2LITE_WAL_AWS_ENDPOINT_URL_S3")?, - region: get("S2LITE_WAL_AWS_REGION")?, - access_key_id: get("S2LITE_WAL_AWS_ACCESS_KEY_ID")?, - secret_access_key: get("S2LITE_WAL_AWS_SECRET_ACCESS_KEY")?, - session_token: get("S2LITE_WAL_AWS_SESSION_TOKEN")?, - }; - eyre::ensure!( - config.access_key_id.is_some() == config.secret_access_key.is_some(), - "S2LITE_WAL_AWS_ACCESS_KEY_ID and S2LITE_WAL_AWS_SECRET_ACCESS_KEY must be set together" - ); - eyre::ensure!( - config.session_token.is_none() || config.access_key_id.is_some(), - "S2LITE_WAL_AWS_SESSION_TOKEN requires the WAL access key and secret key" - ); - Ok(config) - } - - fn apply( - self, - mut builder: object_store::aws::AmazonS3Builder, - ) -> object_store::aws::AmazonS3Builder { - if let Some(endpoint) = &self.endpoint { - // Override the S3-specific endpoint inherited from the main store. - builder = builder - .with_allow_http(endpoint.starts_with("http://")) - .with_config(object_store::aws::AmazonS3ConfigKey::S3Endpoint, endpoint); - } - if let Some(region) = self.region { - builder = builder.with_region(region); - } - if let (Some(key_id), Some(secret_key)) = (self.access_key_id, self.secret_access_key) { - builder = builder.with_credentials(Arc::new( - object_store::StaticCredentialProvider::new(object_store::aws::AwsCredential { - key_id, - secret_key, - token: self.session_token, - }), - )); - } - builder - } + }) } async fn shutdown_signal(handle: axum_server::Handle) { @@ -591,33 +447,7 @@ impl object_store::CredentialProvider for S3CredentialProvider { #[cfg(test)] mod tests { - use super::{ServerProtocol, WalS3Overrides, cli_endpoint, cli_env_hint}; - - fn wal_config(values: &[(&str, &str)]) -> eyre::Result { - WalS3Overrides::from_getter(|name| { - Ok(values - .iter() - .find(|(key, _)| *key == name) - .map(|(_, value)| (*value).to_owned())) - }) - } - - #[test] - fn wal_static_credentials_must_be_complete() { - for values in [ - vec![("S2LITE_WAL_AWS_ACCESS_KEY_ID", "test-key")], - vec![("S2LITE_WAL_AWS_SECRET_ACCESS_KEY", "do-not-log-this-secret")], - vec![("S2LITE_WAL_AWS_SESSION_TOKEN", "do-not-log-this-token")], - ] { - let error = wal_config(&values) - .err() - .expect("invalid credentials") - .to_string(); - assert!(error.contains("S2LITE_WAL_AWS_")); - assert!(!error.contains("do-not-log-this")); - } - assert!(wal_config(&[]).is_ok()); - } + use super::{ServerProtocol, cli_endpoint, cli_env_hint}; #[test] fn cli_endpoint_uses_localhost_with_explicit_port() { diff --git a/lite/tests/backend/common/mod.rs b/lite/tests/backend/common/mod.rs index a4bea287..019728e9 100644 --- a/lite/tests/backend/common/mod.rs +++ b/lite/tests/backend/common/mod.rs @@ -3,7 +3,6 @@ use std::pin::Pin; use futures::Stream; use s2_common::{ basin::BasinName, - config::OptionalStreamConfig, encryption::EncryptionSpec, record::StreamPosition, stream::{AppendAck, AppendInput, StreamName}, @@ -31,7 +30,6 @@ pub async fn append( &basin, &stream, encryption.and_then(encryption_key_for_spec), - OptionalStreamConfig::default(), ) .await? .append(input) @@ -53,7 +51,6 @@ where &basin, &stream, encryption.and_then(encryption_key_for_spec), - OptionalStreamConfig::default(), ) .await? .append_session(inputs); diff --git a/lite/tests/backend/common/read.rs b/lite/tests/backend/common/read.rs index 6e3405ba..28cb6b33 100644 --- a/lite/tests/backend/common/read.rs +++ b/lite/tests/backend/common/read.rs @@ -3,7 +3,6 @@ use std::{pin::Pin, task::Poll, time::Duration}; use futures::StreamExt; use s2_common::{ basin::BasinName, - config::OptionalStreamConfig, encryption::EncryptionSpec, read_extent::{ReadLimit, ReadUntil}, record::{Record, SequencedRecord}, @@ -84,12 +83,7 @@ pub async fn try_open_read_session_with_encryption( ReadError, > { let read_session = backend - .open_for_read( - basin, - stream, - encryption_key_for_spec(encryption), - OptionalStreamConfig::default(), - ) + .open_for_read(basin, stream, encryption_key_for_spec(encryption)) .await? .read(start, end) .await?; diff --git a/lite/tests/backend/common/setup.rs b/lite/tests/backend/common/setup.rs index 597f8023..324ea3b6 100644 --- a/lite/tests/backend/common/setup.rs +++ b/lite/tests/backend/common/setup.rs @@ -1,4 +1,4 @@ -use std::{fmt::Debug, future::Future, sync::Arc, time::Duration}; +use std::{sync::Arc, time::Duration}; use bytes::Bytes; use bytesize::ByteSize; @@ -20,16 +20,12 @@ const TEST_AEGIS256_KEY: [u8; 32] = [0x42; 32]; const TEST_AES256_GCM_KEY: [u8; 32] = [0x24; 32]; pub async fn create_in_memory_db() -> Db { - create_in_memory_db_with_flush_interval(Some(Duration::from_millis(5))).await -} - -async fn create_in_memory_db_with_flush_interval(flush_interval: Option) -> Db { let object_store = Arc::new(InMemory::new()); let db_path = format!("/tmp/test_{}", Uuid::new_v4()); Db::builder(db_path, object_store) .with_settings(Settings { - flush_interval, + flush_interval: Some(Duration::from_millis(5)), ..Default::default() }) .build() @@ -42,50 +38,6 @@ pub async fn create_backend() -> Backend { Backend::new(db, ByteSize::mib(10)) } -pub async fn create_backend_without_auto_flush() -> (Backend, Db) { - let db = create_in_memory_db_with_flush_interval(None).await; - (Backend::new(db.clone(), ByteSize::mib(10)), db) -} - -/// Poll an operation until the database has a committed, unflushed write. -/// The database must have automatic flushing disabled and no unrelated writers. -pub async fn assert_pending_until_committed( - db: &Db, - operation: &mut (impl Future + Unpin), -) -> u64 { - tokio::time::timeout(Duration::from_secs(5), async { - tokio::select! { - biased; - result = operation => panic!("metadata acknowledged before flush: {result:?}"), - seq = async { - loop { - let seq = db.snapshot().await.unwrap().seq(); - if seq > db.status().durable_seq { - return seq; - } - tokio::task::yield_now().await; - } - } => seq, - } - }) - .await - .expect("metadata should be committed in memory before flushing") -} - -pub async fn assert_waits_for_flush(db: &Db, operation: impl Future) -> T { - tokio::pin!(operation); - tokio::time::timeout(Duration::from_secs(5), async { - let seq = assert_pending_until_committed(db, &mut operation).await; - assert!(futures::poll!(&mut operation).is_pending()); - db.flush().await.unwrap(); - let result = operation.await; - assert!(db.status().durable_seq >= seq); - result - }) - .await - .expect("metadata write should finish after an explicit flush") -} - pub fn test_basin_name(suffix: &str) -> BasinName { format!("test-basin-{}", suffix).parse().unwrap() } @@ -285,12 +237,7 @@ pub async fn append_payloads_with_encryption( fencing_token: None, }; backend - .open_for_append( - basin, - stream, - encryption_key_for_spec(encryption), - OptionalStreamConfig::default(), - ) + .open_for_append(basin, stream, encryption_key_for_spec(encryption)) .await .expect("Failed to open append handle") .append(input) @@ -310,7 +257,7 @@ pub async fn append_timestamped_payloads( fencing_token: None, }; backend - .open_for_append(basin, stream, None, OptionalStreamConfig::default()) + .open_for_append(basin, stream, None) .await .expect("Failed to open append handle") .append(input) diff --git a/lite/tests/backend/control_plane/basin.rs b/lite/tests/backend/control_plane/basin.rs index 67e1037d..f5d09f22 100644 --- a/lite/tests/backend/control_plane/basin.rs +++ b/lite/tests/backend/control_plane/basin.rs @@ -4,7 +4,8 @@ use s2_common::{ basin::{BasinNamePrefix, BasinNameStartAfter, ListBasinsRequest}, config::{ BasinConfig, BasinReconfiguration, OptionalDeleteOnEmptyConfig, OptionalStreamConfig, - RetentionPolicy, StreamReconfiguration, TimestampingMode, TimestampingReconfiguration, + RetentionPolicy, StorageClass, StreamReconfiguration, TimestampingMode, + TimestampingReconfiguration, }, maybe::Maybe, resources::{ProvisionMode, ProvisionResult, RequestToken}, @@ -15,80 +16,6 @@ use s2_lite::backend::error::{ use super::common::*; -#[tokio::test] -async fn test_provision_basin_acknowledges_only_durable_metadata() { - let (backend, db) = create_backend_without_auto_flush().await; - let basin = test_basin_name("durable"); - let result = assert_waits_for_flush( - &db, - backend.provision_basin(basin.clone(), BasinConfig::default(), ProvisionMode::Ensure), - ) - .await - .unwrap(); - assert!(matches!(result, ProvisionResult::Created(_))); - assert_eq!( - backend.get_basin_config(basin).await.unwrap(), - BasinConfig::default() - ); - backend.close().await.unwrap(); -} - -#[rstest::rstest] -#[case::ensure(ProvisionMode::Ensure, false)] -#[case::idempotent_create(ProvisionMode::CreateOnly { - request_token: Some("original".parse().unwrap()), -}, false)] -#[case::create_without_token(ProvisionMode::CreateOnly { - request_token: None, -}, true)] -#[case::create_with_different_token(ProvisionMode::CreateOnly { - request_token: Some("different".parse().unwrap()), -}, true)] -#[tokio::test] -async fn test_basin_retries_acknowledge_only_durable_metadata( - #[case] retry_mode: ProvisionMode, - #[case] expect_already_exists: bool, -) { - let (backend, db) = create_backend_without_auto_flush().await; - let basin = test_basin_name("durable-retry"); - let creation = backend.provision_basin( - basin.clone(), - BasinConfig::default(), - ProvisionMode::CreateOnly { - request_token: Some("original".parse().unwrap()), - }, - ); - tokio::pin!(creation); - assert_pending_until_committed(&db, &mut creation).await; - assert!(matches!( - backend.get_basin_config(basin.clone()).await, - Err(GetBasinConfigError::BasinNotFound(_)) - )); - - let retry = assert_waits_for_flush( - &db, - backend.provision_basin(basin.clone(), BasinConfig::default(), retry_mode), - ) - .await; - if expect_already_exists { - assert!(matches!( - retry, - Err(ProvisionBasinError::BasinAlreadyExists(_)) - )); - } else { - assert!(matches!(retry, Ok(ProvisionResult::Noop(_)))); - } - assert!(matches!( - creation.await.unwrap(), - ProvisionResult::Created(_) - )); - assert_eq!( - backend.get_basin_config(basin).await.unwrap(), - BasinConfig::default() - ); - backend.close().await.unwrap(); -} - #[tokio::test] async fn test_create_basin_idempotency_respects_request_token() { let backend = create_backend().await; @@ -240,7 +167,7 @@ async fn test_provision_basin_ensure_updates_config() { let mut updated_config = initial_config.clone(); updated_config.create_stream_on_append = true; updated_config.create_stream_on_read = true; - updated_config.default_stream_config.storage_class = Some("standard".into()); + updated_config.default_stream_config.storage_class = Some(StorageClass::Standard); backend .provision_basin( @@ -259,7 +186,7 @@ async fn test_provision_basin_ensure_updates_config() { assert!(stored_config.create_stream_on_read); assert_eq!( stored_config.default_stream_config.storage_class, - Some("standard".into()) + Some(StorageClass::Standard) ); backend @@ -282,7 +209,7 @@ async fn test_provision_basin_ensure_resets_unspecified_config() { create_stream_on_append: true, create_stream_on_read: false, default_stream_config: OptionalStreamConfig { - storage_class: Some("standard".into()), + storage_class: Some(StorageClass::Standard), retention_policy: Some(RetentionPolicy::Infinite()), ..Default::default() }, @@ -369,7 +296,7 @@ async fn test_reconfigure_basin_updates_nested_defaults() { let backend = create_backend().await; let basin_name = test_basin_name("basin-reconfigure"); let mut initial_config = BasinConfig::default(); - initial_config.default_stream_config.storage_class = Some("standard".into()); + initial_config.default_stream_config.storage_class = Some(StorageClass::Standard); backend .provision_basin( @@ -387,7 +314,7 @@ async fn test_reconfigure_basin_updates_nested_defaults() { ..Default::default() }; let mut stream_reconfig = StreamReconfiguration { - storage_class: Maybe::from(Some("express".into())), + storage_class: Maybe::from(Some(StorageClass::Express)), retention_policy: Maybe::from(Some(RetentionPolicy::Infinite())), ..Default::default() }; @@ -409,7 +336,7 @@ async fn test_reconfigure_basin_updates_nested_defaults() { assert!(updated.create_stream_on_read); assert_eq!( updated.default_stream_config.storage_class, - Some("express".into()) + Some(StorageClass::Express) ); assert_eq!( updated.default_stream_config.retention_policy, @@ -426,7 +353,7 @@ async fn test_reconfigure_basin_updates_nested_defaults() { .expect("Failed to fetch basin config after reconfigure"); assert_eq!( fetched.default_stream_config.storage_class, - Some("express".into()) + Some(StorageClass::Express) ); assert_eq!( fetched.default_stream_config.retention_policy, @@ -440,30 +367,6 @@ async fn test_reconfigure_basin_updates_nested_defaults() { assert!(fetched.create_stream_on_read); } -#[tokio::test(start_paused = true)] -async fn test_delete_basin_retry_waits_for_durable_metadata() { - let (backend, db) = create_backend_without_auto_flush().await; - let basin = assert_waits_for_flush( - &db, - create_test_basin(&backend, "durable-delete", BasinConfig::default()), - ) - .await; - - let mut first = Box::pin(backend.delete_basin(basin.clone())); - assert_pending_until_committed(&db, &mut first).await; - drop(first); - - let retry = backend.delete_basin(basin); - tokio::pin!(retry); - assert!( - tokio::time::timeout(Duration::from_secs(1), &mut retry) - .await - .is_err() - ); - assert_waits_for_flush(&db, retry).await.unwrap(); - backend.close().await.unwrap(); -} - #[tokio::test] async fn test_delete_basin_marks_deleting_and_blocks_create() { let backend = create_backend().await; diff --git a/lite/tests/backend/control_plane/stream.rs b/lite/tests/backend/control_plane/stream.rs index 6ded3547..a3ca62c5 100644 --- a/lite/tests/backend/control_plane/stream.rs +++ b/lite/tests/backend/control_plane/stream.rs @@ -5,7 +5,8 @@ use s2_common::{ config::{ BasinConfig, BasinReconfiguration, DeleteOnEmptyReconfiguration, OptionalDeleteOnEmptyConfig, OptionalStreamConfig, OptionalTimestampingConfig, - RetentionPolicy, StreamReconfiguration, TimestampingMode, TimestampingReconfiguration, + RetentionPolicy, StorageClass, StreamReconfiguration, TimestampingMode, + TimestampingReconfiguration, }, encryption::EncryptionAlgorithm, maybe::Maybe, @@ -22,34 +23,6 @@ use s2_lite::backend::error::{ use super::common::*; -#[tokio::test] -async fn test_provision_stream_acknowledges_only_durable_metadata() { - let (backend, db) = create_backend_without_auto_flush().await; - let basin = test_basin_name("durable-stream"); - let stream = test_stream_name("durable"); - assert_waits_for_flush( - &db, - backend.provision_basin(basin.clone(), BasinConfig::default(), ProvisionMode::Ensure), - ) - .await - .unwrap(); - - let result = assert_waits_for_flush( - &db, - backend.provision_stream( - basin.clone(), - stream.clone(), - OptionalStreamConfig::default(), - ProvisionMode::Ensure, - ), - ) - .await - .unwrap(); - assert!(matches!(result, ProvisionResult::Created(_))); - backend.get_stream_config(basin, stream).await.unwrap(); - backend.close().await.unwrap(); -} - #[tokio::test] async fn test_create_stream_honors_basin_defaults() { let backend = create_backend().await; @@ -57,7 +30,7 @@ async fn test_create_stream_honors_basin_defaults() { let basin_config = BasinConfig { default_stream_config: OptionalStreamConfig { - storage_class: Some("standard".into()), + storage_class: Some(StorageClass::Standard), retention_policy: Some(RetentionPolicy::Infinite()), timestamping: OptionalTimestampingConfig { mode: Some(TimestampingMode::ClientRequire), @@ -97,7 +70,7 @@ async fn test_create_stream_honors_basin_defaults() { .get_stream_config(basin_name, stream_name) .await .expect("Failed to fetch stream config"); - assert_eq!(config.storage_class.as_deref(), Some("standard")); + assert_eq!(config.storage_class, StorageClass::Standard); assert_eq!(config.retention_policy, RetentionPolicy::Infinite()); assert_eq!(config.timestamping.mode, TimestampingMode::ClientRequire); } @@ -239,7 +212,7 @@ async fn test_create_stream_idempotency_and_request_token() { let stream_name = test_stream_name("stream-idempotency"); let config = OptionalStreamConfig { - storage_class: Some("express".into()), + storage_class: Some(StorageClass::Express), ..Default::default() }; @@ -261,7 +234,7 @@ async fn test_create_stream_idempotency_and_request_token() { .get_stream_config(basin_name.clone(), stream_name.clone()) .await .expect("Failed to fetch stored stream config"); - assert_eq!(stored_config.storage_class.as_deref(), Some("express")); + assert_eq!(stored_config.storage_class, StorageClass::Express); let idempotent = backend .provision_stream( @@ -325,7 +298,7 @@ async fn test_provision_stream_ensure_preserves_idempotency_key() { let stream_name = test_stream_name("stream-idempotency-key-preserve"); let config = OptionalStreamConfig { - storage_class: Some("standard".into()), + storage_class: Some(StorageClass::Standard), ..Default::default() }; let token: RequestToken = "stream-token-preserve".parse().unwrap(); @@ -374,7 +347,7 @@ async fn test_provision_stream_ensure_preserves_idempotency_key() { .get_stream_config(basin_name.clone(), stream_name.clone()) .await .expect("Failed to fetch stream config"); - assert_eq!(stored_config.storage_class.as_deref(), Some("express")); + assert_eq!(stored_config.storage_class, StorageClass::Express); assert_eq!(stored_config.timestamping.mode, TimestampingMode::Arrival); backend @@ -398,7 +371,7 @@ async fn test_provision_stream_ensure_noops_when_effective_config_matches() { "stream-ensure-effective-noop", BasinConfig { default_stream_config: OptionalStreamConfig { - storage_class: Some("express".into()), + storage_class: Some(StorageClass::Express), retention_policy: Some(RetentionPolicy::Age(Duration::from_secs( 10 * 24 * 60 * 60, ))), @@ -410,7 +383,7 @@ async fn test_provision_stream_ensure_noops_when_effective_config_matches() { .await; let stream_name = test_stream_name("stream-ensure-effective-noop"); let config = OptionalStreamConfig { - storage_class: Some("standard".into()), + storage_class: Some(StorageClass::Standard), retention_policy: Some(RetentionPolicy::Infinite()), ..Default::default() }; @@ -516,7 +489,7 @@ async fn test_provision_stream_idempotency_ignores_changed_basin_defaults() { basin_name.clone(), BasinReconfiguration { default_stream_config: Maybe::from(Some(StreamReconfiguration { - storage_class: Maybe::from(Some("standard".into())), + storage_class: Maybe::from(Some(StorageClass::Standard)), ..Default::default() })), ..Default::default() @@ -544,7 +517,7 @@ async fn test_reconfigure_stream_updates_selected_fields() { let basin_name = test_basin_name("stream-reconfigure"); let mut basin_config = BasinConfig::default(); - basin_config.default_stream_config.storage_class = Some("standard".into()); + basin_config.default_stream_config.storage_class = Some(StorageClass::Standard); backend .provision_basin( @@ -584,7 +557,7 @@ async fn test_reconfigure_stream_updates_selected_fields() { uncapped: Maybe::from(Some(true)), }; let mut stream_reconfig = StreamReconfiguration { - storage_class: Maybe::from(Some("express".into())), + storage_class: Maybe::from(Some(StorageClass::Express)), retention_policy: Maybe::from(Some(RetentionPolicy::Infinite())), ..Default::default() }; @@ -595,7 +568,7 @@ async fn test_reconfigure_stream_updates_selected_fields() { .await .expect("Failed to reconfigure stream"); - assert_eq!(updated.storage_class.as_deref(), Some("express")); + assert_eq!(updated.storage_class, StorageClass::Express); assert_eq!(updated.retention_policy, RetentionPolicy::Infinite()); assert_eq!(updated.timestamping.mode, TimestampingMode::Arrival); assert!(updated.timestamping.uncapped); @@ -604,7 +577,7 @@ async fn test_reconfigure_stream_updates_selected_fields() { .get_stream_config(basin_name, stream_name) .await .expect("Failed to fetch stream config after reconfigure"); - assert_eq!(fetched.storage_class.as_deref(), Some("express")); + assert_eq!(fetched.storage_class, StorageClass::Express); assert_eq!(fetched.retention_policy, RetentionPolicy::Infinite()); assert_eq!(fetched.timestamping.mode, TimestampingMode::Arrival); assert!(fetched.timestamping.uncapped); @@ -617,7 +590,7 @@ async fn test_reconfigure_stream_clears_fields_to_basin_defaults() { let basin_config = BasinConfig { default_stream_config: OptionalStreamConfig { - storage_class: Some("standard".into()), + storage_class: Some(StorageClass::Standard), retention_policy: Some(RetentionPolicy::Infinite()), timestamping: OptionalTimestampingConfig { mode: Some(TimestampingMode::Arrival), @@ -643,7 +616,7 @@ async fn test_reconfigure_stream_clears_fields_to_basin_defaults() { let stream_name = test_stream_name("stream-reconfigure-clear-defaults"); let stream_config = OptionalStreamConfig { - storage_class: Some("express".into()), + storage_class: Some(StorageClass::Express), retention_policy: Some(RetentionPolicy::Age(Duration::from_secs(60))), timestamping: OptionalTimestampingConfig { mode: Some(TimestampingMode::ClientRequire), @@ -683,7 +656,7 @@ async fn test_reconfigure_stream_clears_fields_to_basin_defaults() { .await .expect("Failed to reconfigure stream"); - assert_eq!(updated.storage_class.as_deref(), Some("standard")); + assert_eq!(updated.storage_class, StorageClass::Standard); assert_eq!(updated.retention_policy, RetentionPolicy::Infinite()); assert_eq!(updated.timestamping.mode, TimestampingMode::Arrival); assert!(updated.timestamping.uncapped); @@ -697,101 +670,84 @@ async fn test_reconfigure_stream_clears_fields_to_basin_defaults() { assert_eq!(fetched, updated); } -#[rstest::rstest] -#[case::patch(false, false)] -#[case::ensure(true, false)] -#[case::cancelled_patch(false, true)] -#[case::cancelled_ensure(true, true)] #[tokio::test] -async fn test_stream_config_updates_active_streamer(#[case] ensure: bool, #[case] cancel: bool) { - let (backend, db) = create_backend_without_auto_flush().await; - let basin = assert_waits_for_flush( - &db, - create_test_basin(&backend, "config-delivery", BasinConfig::default()), +async fn test_reconfigure_stream_updates_active_streamer() { + let (backend, basin_name, stream_name) = setup_backend_with_stream( + "stream-reconfigure-active", + "stream", + OptionalStreamConfig::default(), ) .await; - let stream = assert_waits_for_flush( - &db, - create_test_stream( - &backend, - &basin, - "stream", - OptionalStreamConfig { - timestamping: OptionalTimestampingConfig { - mode: Some(TimestampingMode::ClientRequire), - ..Default::default() - }, - ..Default::default() - }, - ), - ) - .await; - let _active_streamer = backend - .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) - .await - .unwrap(); - let mut update = Box::pin(async { - if ensure { - backend - .provision_stream( - basin.clone(), - stream.clone(), - OptionalStreamConfig { - timestamping: OptionalTimestampingConfig { - mode: Some(TimestampingMode::Arrival), - ..Default::default() - }, - ..Default::default() - }, - ProvisionMode::Ensure, - ) - .await - .unwrap(); - } else { - backend - .reconfigure_stream( - basin.clone(), - stream.clone(), - StreamReconfiguration { - timestamping: Maybe::from(Some(TimestampingReconfiguration { - mode: Maybe::from(Some(TimestampingMode::Arrival)), - ..Default::default() - })), - ..Default::default() - }, - ) - .await - .unwrap(); - } - }); - assert_pending_until_committed(&db, &mut update).await; - if cancel { - drop(update); - db.flush().await.unwrap(); - } else { - db.flush().await.unwrap(); - update.await; - } + append_payloads(&backend, &basin_name, &stream_name, &[b"seed"]).await; - // Application is asynchronous; retry until the active streamer accepts arrival timestamps. - let append_after_update = async { - loop { - let input = AppendInput { - records: create_test_record_batch(vec![Bytes::from_static(b"no timestamp")]), - match_seq_num: None, - fencing_token: None, - }; - match append(&backend, basin.clone(), stream.clone(), input, None).await { - Err(AppendError::TimestampMissing(_)) => tokio::task::yield_now().await, - result => break result, - } - } + let ts_reconfig = TimestampingReconfiguration { + mode: Maybe::from(Some(TimestampingMode::ClientRequire)), + uncapped: Maybe::default(), }; - assert_waits_for_flush(&db, append_after_update) + let reconfig = StreamReconfiguration { + timestamping: Maybe::from(Some(ts_reconfig)), + ..Default::default() + }; + + backend + .reconfigure_stream(basin_name.clone(), stream_name.clone(), reconfig) .await - .unwrap(); - backend.close().await.unwrap(); + .expect("Failed to reconfigure stream"); + + check_tail(&backend, basin_name.clone(), stream_name.clone()) + .await + .expect("Failed to check tail"); + + let input = AppendInput { + records: create_test_record_batch(vec![Bytes::from_static(b"missing timestamp")]), + match_seq_num: None, + fencing_token: None, + }; + let result = append(&backend, basin_name, stream_name, input, None).await; + assert!(matches!(result, Err(AppendError::TimestampMissing(_)))); +} + +#[tokio::test] +async fn test_provision_stream_ensure_updates_active_streamer() { + let (backend, basin_name, stream_name) = setup_backend_with_stream( + "stream-ensure-active", + "stream", + OptionalStreamConfig::default(), + ) + .await; + + append_payloads(&backend, &basin_name, &stream_name, &[b"seed"]).await; + + let config = OptionalStreamConfig { + timestamping: OptionalTimestampingConfig { + mode: Some(TimestampingMode::ClientRequire), + ..Default::default() + }, + ..Default::default() + }; + + backend + .provision_stream( + basin_name.clone(), + stream_name.clone(), + config, + ProvisionMode::Ensure, + ) + .await + .expect("Ensure should succeed for an existing stream"); + + check_tail(&backend, basin_name.clone(), stream_name.clone()) + .await + .expect("Failed to check tail"); + + let input = AppendInput { + records: create_test_record_batch(vec![Bytes::from_static(b"missing timestamp")]), + match_seq_num: None, + fencing_token: None, + }; + let result = append(&backend, basin_name, stream_name, input, None).await; + assert!(matches!(result, Err(AppendError::TimestampMissing(_)))); } #[tokio::test] @@ -823,38 +779,6 @@ async fn test_create_stream_fails_when_basin_deleting() { )); } -#[tokio::test(start_paused = true)] -async fn test_delete_stream_retry_waits_for_durable_metadata() { - let (backend, db) = create_backend_without_auto_flush().await; - let basin = assert_waits_for_flush( - &db, - create_test_basin(&backend, "durable-delete", BasinConfig::default()), - ) - .await; - let stream = assert_waits_for_flush( - &db, - create_test_stream(&backend, &basin, "durable-delete", Default::default()), - ) - .await; - - let mut first = Box::pin(backend.delete_stream(basin.clone(), stream.clone())); - // Stream deletion first persists the terminal trim, then the metadata marker. - assert_pending_until_committed(&db, &mut first).await; - db.flush().await.unwrap(); - assert_pending_until_committed(&db, &mut first).await; - drop(first); - - let retry = backend.delete_stream(basin, stream); - tokio::pin!(retry); - assert!( - tokio::time::timeout(Duration::from_secs(1), &mut retry) - .await - .is_err() - ); - assert_waits_for_flush(&db, retry).await.unwrap(); - backend.close().await.unwrap(); -} - #[tokio::test] async fn test_delete_stream_marks_deleted_and_blocks_recreation() { let backend = create_backend().await; diff --git a/lite/tests/backend/data_plane/auto_create.rs b/lite/tests/backend/data_plane/auto_create.rs index 491a3eb7..24bdfc00 100644 --- a/lite/tests/backend/data_plane/auto_create.rs +++ b/lite/tests/backend/data_plane/auto_create.rs @@ -3,10 +3,7 @@ use std::time::Duration; use bytes::Bytes; use s2_common::{ basin::BasinName, - config::{ - BasinConfig, DeleteOnEmptyConfig, OptionalDeleteOnEmptyConfig, OptionalStreamConfig, - RetentionPolicy, StreamConfig, - }, + config::BasinConfig, encryption::EncryptionAlgorithm, read_extent::{ReadLimit, ReadUntil}, record::StreamPosition, @@ -126,140 +123,6 @@ async fn test_backend_append_auto_creates_stream_with_basin_cipher() { assert_eq!(envelope_bodies(&records), vec![b"secret".to_vec()]); } -fn basin_config_with_defaults() -> BasinConfig { - BasinConfig { - create_stream_on_append: true, - default_stream_config: OptionalStreamConfig { - storage_class: Some("standard".into()), - retention_policy: Some(RetentionPolicy::Age(Duration::from_secs(7 * 24 * 60 * 60))), - ..Default::default() - }, - ..Default::default() - } -} - -fn requested_stream_config() -> OptionalStreamConfig { - OptionalStreamConfig { - retention_policy: Some(RetentionPolicy::Age(Duration::from_secs(3600))), - delete_on_empty: OptionalDeleteOnEmptyConfig { - min_age: Some(Duration::from_secs(300)), - }, - ..Default::default() - } -} - -fn expected_merged_stream_config() -> StreamConfig { - StreamConfig { - storage_class: Some("standard".into()), - retention_policy: RetentionPolicy::Age(Duration::from_secs(3600)), - timestamping: Default::default(), - delete_on_empty: DeleteOnEmptyConfig { - min_age: Duration::from_secs(300), - }, - } -} - -#[tokio::test] -async fn test_backend_append_auto_create_applies_stream_config() { - let backend = create_backend().await; - let basin_name = create_test_basin( - &backend, - "backend-auto-create-config", - basin_config_with_defaults(), - ) - .await; - let stream_name = test_stream_name("missing"); - - let input = AppendInput { - records: create_test_record_batch(vec![Bytes::from_static(b"hello")]), - match_seq_num: None, - fencing_token: None, - }; - let ack = backend - .open_for_append(&basin_name, &stream_name, None, requested_stream_config()) - .await - .expect("Failed to open append handle") - .append(input) - .await - .expect("Failed to append to auto-created stream"); - assert_eq!(ack.end.seq_num, 1); - - let config = backend - .get_stream_config(basin_name.clone(), stream_name.clone()) - .await - .expect("Failed to get stream config"); - assert_eq!(config, expected_merged_stream_config()); -} - -#[tokio::test] -async fn test_backend_append_ignores_stream_config_for_existing_stream() { - let backend = create_backend().await; - let basin_name = create_test_basin( - &backend, - "backend-auto-create-config-existing", - basin_config_with_defaults(), - ) - .await; - let stream_name = create_test_stream( - &backend, - &basin_name, - "existing", - OptionalStreamConfig::default(), - ) - .await; - let before = backend - .get_stream_config(basin_name.clone(), stream_name.clone()) - .await - .expect("Failed to get stream config"); - - let input = AppendInput { - records: create_test_record_batch(vec![Bytes::from_static(b"hello")]), - match_seq_num: None, - fencing_token: None, - }; - backend - .open_for_append(&basin_name, &stream_name, None, requested_stream_config()) - .await - .expect("Failed to open append handle") - .append(input) - .await - .expect("Failed to append to existing stream"); - - let after = backend - .get_stream_config(basin_name.clone(), stream_name.clone()) - .await - .expect("Failed to get stream config"); - assert_eq!(after, before); - assert_ne!(after, expected_merged_stream_config()); -} - -#[tokio::test] -async fn test_backend_read_auto_create_applies_stream_config() { - let backend = create_backend().await; - let basin_name = create_test_basin( - &backend, - "backend-auto-create-read-config", - BasinConfig { - create_stream_on_append: false, - create_stream_on_read: true, - ..basin_config_with_defaults() - }, - ) - .await; - let stream_name = test_stream_name("missing"); - - backend - .open_for_read(&basin_name, &stream_name, None, requested_stream_config()) - .await - .expect("Failed to open read handle on auto-created stream"); - - let config = backend - .get_stream_config(basin_name.clone(), stream_name.clone()) - .await - .expect("Failed to get stream config"); - assert_eq!(config, expected_merged_stream_config()); -} - #[tokio::test] async fn test_backend_append_without_auto_create_returns_not_found() { let backend = create_backend().await; diff --git a/lite/tests/backend/data_plane/mixed.rs b/lite/tests/backend/data_plane/mixed.rs index 7bc8c40c..63b9a372 100644 --- a/lite/tests/backend/data_plane/mixed.rs +++ b/lite/tests/backend/data_plane/mixed.rs @@ -2,7 +2,7 @@ use std::{sync::Arc, time::Duration}; use bytes::Bytes; use s2_common::{ - config::{OptionalStreamConfig, RetentionPolicy, StreamReconfiguration}, + config::{OptionalStreamConfig, RetentionPolicy, StorageClass, StreamReconfiguration}, read_extent::{ReadLimit, ReadUntil}, stream::{AppendInput, ReadEnd, ReadFrom, ReadStart}, }; @@ -147,7 +147,7 @@ async fn test_concurrent_reconfigure_during_append() { ready.notified().await; let reconfig = StreamReconfiguration { - storage_class: s2_common::maybe::Maybe::from(Some("express".into())), + storage_class: s2_common::maybe::Maybe::from(Some(StorageClass::Express)), retention_policy: s2_common::maybe::Maybe::from(Some(RetentionPolicy::Infinite())), timestamping: s2_common::maybe::Maybe::default(), delete_on_empty: s2_common::maybe::Maybe::default(), @@ -157,7 +157,7 @@ async fn test_concurrent_reconfigure_during_append() { .reconfigure_stream(basin_name.clone(), stream_name.clone(), reconfig) .await .expect("Failed to reconfigure stream during appends"); - assert_eq!(updated_config.storage_class.as_deref(), Some("express")); + assert_eq!(updated_config.storage_class, StorageClass::Express); append_handle.await.unwrap(); diff --git a/lite/tests/backend/data_plane/read.rs b/lite/tests/backend/data_plane/read.rs index 511ae50c..900d89ee 100644 --- a/lite/tests/backend/data_plane/read.rs +++ b/lite/tests/backend/data_plane/read.rs @@ -326,31 +326,6 @@ async fn test_read_at_tail_without_follow_returns_unwritten( } } -#[rstest] -#[case::seq_num(ReadFrom::SeqNum(0))] -#[case::timestamp(ReadFrom::Timestamp(0))] -#[tokio::test] -async fn test_read_at_tail_of_empty_stream_returns_unwritten(#[case] from: ReadFrom) { - let (backend, basin_name, stream_name) = setup_backend_with_stream( - "read-empty-at-tail", - "stream", - OptionalStreamConfig::default(), - ) - .await; - - let start = ReadStart { from, clamp: false }; - let end = tail_read_end(TailEndCase::CountNoWait); - let result = try_open_read_session(&backend, &basin_name, &stream_name, start, end).await; - - match result { - Err(ReadError::Unwritten(UnwrittenError(tail))) => { - assert_eq!(tail, StreamPosition::MIN); - } - Ok(_) => panic!("Expected Unwritten error for {from:?} on an empty stream, got Ok"), - Err(e) => panic!("Expected Unwritten error for {from:?} on an empty stream, got: {e:?}"), - } -} - #[tokio::test] async fn test_read_from_tail_offset() { let (backend, basin_name, stream_name) = setup_backend_with_stream( diff --git a/resource-spec/CHANGELOG.md b/resource-spec/CHANGELOG.md index d5368b6d..745d023e 100644 --- a/resource-spec/CHANGELOG.md +++ b/resource-spec/CHANGELOG.md @@ -2,14 +2,6 @@ All notable changes to this project will be documented in this file. -## [0.3.0] - 2026-09-25 - -### Features - -- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) - - - ## [0.2.2] - 2026-07-07 ### Miscellaneous Tasks diff --git a/resource-spec/Cargo.toml b/resource-spec/Cargo.toml index 4c5816c3..1c0ca47d 100644 --- a/resource-spec/Cargo.toml +++ b/resource-spec/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-resource-spec" -version = "0.3.0" +version = "0.2.2" description = "Declarative resource specifications for S2" edition.workspace = true license.workspace = true @@ -9,7 +9,6 @@ homepage.workspace = true keywords = ["s2", "durable", "streams", "config", "schema"] [dependencies] -compact_str = { workspace = true, features = ["serde"] } humantime = { workspace = true } s2-common = { workspace = true } schemars = { workspace = true } diff --git a/resource-spec/src/lib.rs b/resource-spec/src/lib.rs index 042edd6c..7a0ea7fa 100644 --- a/resource-spec/src/lib.rs +++ b/resource-spec/src/lib.rs @@ -2,7 +2,6 @@ use std::{borrow::Cow, time::Duration}; -use compact_str::CompactString; use s2_common::{basin::BasinName, stream::StreamName}; use serde::{Deserialize, Serialize}; @@ -51,10 +50,9 @@ pub struct BasinConfig { #[derive(Debug, Clone, Deserialize, Default, schemars::JsonSchema)] #[serde(deny_unknown_fields)] pub struct StreamConfig { - /// [Storage class](https://s2.dev/docs/storage-classes) for recent writes. + /// Storage class for recent writes. #[serde(default)] - #[schemars(with = "Option")] - pub storage_class: Option, + pub storage_class: Option, /// Retention policy for the stream. If unspecified, the default is to retain records for 7 /// days. #[serde(default)] @@ -67,6 +65,36 @@ pub struct StreamConfig { pub delete_on_empty: Option, } +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum StorageClass { + Standard, + Express, +} + +impl schemars::JsonSchema for StorageClass { + fn schema_name() -> Cow<'static, str> { + "StorageClass".into() + } + + fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { + schemars::json_schema!({ + "type": "string", + "description": "Storage class for recent writes.", + "enum": ["standard", "express"] + }) + } +} + +impl From for s2_common::config::StorageClass { + fn from(s: StorageClass) -> Self { + match s { + StorageClass::Standard => Self::Standard, + StorageClass::Express => Self::Express, + } + } +} + #[derive(Debug, Clone, Deserialize, Serialize)] pub enum EncryptionAlgorithm { #[serde(rename = "aegis-256")] @@ -259,7 +287,7 @@ impl From for s2_common::config::OptionalDeleteOnEmptyConfig { impl From for s2_common::config::OptionalStreamConfig { fn from(s: StreamConfig) -> Self { Self { - storage_class: s.storage_class, + storage_class: s.storage_class.map(Into::into), retention_policy: s.retention_policy.map(|rp| rp.0), timestamping: s.timestamping.map(Into::into).unwrap_or_default(), delete_on_empty: s.delete_on_empty.map(Into::into).unwrap_or_default(), @@ -323,9 +351,8 @@ pub fn validate(spec: &Resources) -> Result<(), String> { } fn validate_stream_config(config: &StreamConfig, context: &str, errors: &mut Vec) { - if let Some(retention_policy) = config.retention_policy - && let Err(err) = retention_policy.0.validate() - { + let config = s2_common::config::OptionalStreamConfig::from(config.clone()); + if let Err(err) = config.validate() { errors.push(format!("{context}: {err}")); } } @@ -432,7 +459,7 @@ mod tests { assert_eq!(config.create_stream_on_read, Some(false)); let dsc = config.default_stream_config.as_ref().unwrap(); - assert_eq!(dsc.storage_class.as_deref(), Some("express")); + assert!(matches!(dsc.storage_class, Some(StorageClass::Express))); assert!(matches!( dsc.retention_policy.as_ref().map(|r| &r.0), Some(s2_common::config::RetentionPolicy::Age(_)) @@ -452,7 +479,7 @@ mod tests { let stream = &basin.streams[0]; assert_eq!(stream.name.as_ref(), "events"); let sc = stream.config.as_ref().unwrap(); - assert_eq!(sc.storage_class.as_deref(), Some("standard")); + assert!(matches!(sc.storage_class, Some(StorageClass::Standard))); assert!(matches!( sc.retention_policy.as_ref().map(|r| &r.0), Some(s2_common::config::RetentionPolicy::Infinite()) @@ -577,7 +604,7 @@ mod tests { #[test] fn stream_config_conversion() { let spec = StreamConfig { - storage_class: Some("standard".into()), + storage_class: Some(StorageClass::Standard), retention_policy: Some(RetentionPolicy( s2_common::config::RetentionPolicy::Infinite(), )), @@ -585,7 +612,10 @@ mod tests { delete_on_empty: None, }; let config = s2_common::config::OptionalStreamConfig::from(spec); - assert_eq!(config.storage_class, Some("standard".into())); + assert_eq!( + config.storage_class, + Some(s2_common::config::StorageClass::Standard) + ); assert_eq!( config.retention_policy, Some(s2_common::config::RetentionPolicy::Infinite()) diff --git a/sdk/CHANGELOG.md b/sdk/CHANGELOG.md index ed7c6a3b..b45dbbec 100644 --- a/sdk/CHANGELOG.md +++ b/sdk/CHANGELOG.md @@ -2,71 +2,6 @@ All notable changes to this project will be documented in this file. -## [0.35.0] - 2026-09-25 - -### Features - -- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) - - - -## [0.34.10] - 2026-09-24 - -### Bug Fixes - -- Preserve uncertainty across append retries ([#767](https://github.com/s2-streamstore/s2/issues/767)) - - - -## [0.34.9] - 2026-09-22 - - - -## [0.34.8] - 2026-09-16 - -### Bug Fixes - -- Reject Content-Type in default headers ([#740](https://github.com/s2-streamstore/s2/issues/740)) - - - -## [0.34.7] - 2026-09-11 - -### Features - -- `s2-stream-config` header for auto-created streams ([#718](https://github.com/s2-streamstore/s2/issues/718)) - - - -## [0.34.6] - 2026-09-11 - -### Features - -- Set default request headers (`_hidden` only) ([#731](https://github.com/s2-streamstore/s2/issues/731)) - - - -## [0.34.5] - 2026-09-10 - -### Bug Fixes - -- Reject NUL bytes in stream names and access token IDs ([#728](https://github.com/s2-streamstore/s2/issues/728)) - - - -## [0.34.4] - 2026-09-01 - -### Features - -- Act on s2s reconnect advice in append and read sessions ([#703](https://github.com/s2-streamstore/s2/issues/703)) - -### Bug Fixes - -- Require http2 ([#710](https://github.com/s2-streamstore/s2/issues/710)) -- Classify h2 REFUSED_STREAM as retryable ([#716](https://github.com/s2-streamstore/s2/issues/716)) - - - ## [0.34.3] - 2026-08-13 ### Features diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml index db739f37..4593074f 100644 --- a/sdk/Cargo.toml +++ b/sdk/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "s2-sdk" description = "Rust SDK for S2" -version = "0.35.0" +version = "0.34.3" edition.workspace = true license.workspace = true repository = "https://github.com/s2-streamstore/s2/tree/main/sdk" @@ -25,10 +25,8 @@ async-compression = { version = "0.4", features = ["tokio", "gzip", "zstd"] } async-stream = { workspace = true } async-trait = { workspace = true } bytes = { workspace = true } -compact_str = { workspace = true, features = ["serde"] } futures-core = { workspace = true } futures-util = { workspace = true } -h2 = "0.4" http = { workspace = true } http-body = "1" http-body-util = "0.1" @@ -47,7 +45,7 @@ serde_json = { workspace = true } serde_urlencoded = "0.7" thiserror = { workspace = true } time = { workspace = true } -tokio = { version = "1.53", features = ["time", "macros", "io-util", "sync"] } +tokio = { version = "1.6", features = ["time", "macros", "io-util", "sync"] } tokio-muxt = "0.7" tokio-stream = { workspace = true } tokio-util = { workspace = true, features = ["rt", "codec"] } diff --git a/sdk/examples/reconfigure_basin.rs b/sdk/examples/reconfigure_basin.rs index a1946d28..45a1488d 100644 --- a/sdk/examples/reconfigure_basin.rs +++ b/sdk/examples/reconfigure_basin.rs @@ -1,8 +1,8 @@ use s2_sdk::{ S2, types::{ - BasinName, BasinReconfiguration, ReconfigureBasinInput, S2Config, StreamReconfiguration, - TimestampingReconfiguration, + BasinName, BasinReconfiguration, ReconfigureBasinInput, S2Config, StorageClass, + StreamReconfiguration, TimestampingReconfiguration, }, }; @@ -22,7 +22,7 @@ async fn main() -> Result<(), Box> { BasinReconfiguration::new() .with_default_stream_config( StreamReconfiguration::new() - .with_storage_class("standard") + .with_storage_class(StorageClass::Standard) .with_timestamping(TimestampingReconfiguration::new().with_uncapped(true)), ) .with_create_stream_on_read(true), diff --git a/sdk/src/api.rs b/sdk/src/api.rs index 84175479..2ed7e37a 100644 --- a/sdk/src/api.rs +++ b/sdk/src/api.rs @@ -18,10 +18,7 @@ use s2_api::v1::{ basin::{ BasinInfo, CreateBasinRequest, EnsureBasinRequest, ListBasinsRequest, ListBasinsResponse, }, - config::{ - BasinConfig, BasinReconfiguration, STREAM_CONFIG_HEADER, StreamConfig, - StreamReconfiguration, - }, + config::{BasinConfig, BasinReconfiguration, StreamConfig, StreamReconfiguration}, location::LocationInfo, metrics::{ AccountMetricSetRequest, BasinMetricSetRequest, MetricSetResponse, StreamMetricSetRequest, @@ -45,8 +42,7 @@ use crate::{ client::{self, StreamingResponse, UnaryResponse}, error::{ClientError, server_error_has_no_side_effects, server_error_is_retryable}, frame_signal::FrameSignal, - reconnect::ReconnectAdvice, - retry::{AppendRetryError, RetryBackoff, RetryBackoffBuilder}, + retry::{RetryBackoff, RetryBackoffBuilder}, types::{ AccessToken, AccessTokenId, AccessTokenMode, AppendRetryPolicy, BasinAuthority, BasinName, Compression, EncryptionKey, LocationName, RetryConfig, S2Config, S2Endpoints, StreamName, @@ -387,7 +383,6 @@ impl BasinClient { name: &StreamName, input: AppendInput, encryption: Option<&EncryptionKey>, - stream_config: Option<&StreamConfig>, append_retry_policy: AppendRetryPolicy, ) -> Result { let url = self.uri(format!("v1/streams/{}/records", urlencoding::encode(name))); @@ -398,7 +393,6 @@ impl BasinClient { .body(input.encode_to_vec()) .build()?; set_encryption_header(&mut request, encryption); - set_stream_config_header(&mut request, stream_config); let response = self .request(request) .with_append_retry_policy(append_retry_policy) @@ -425,7 +419,6 @@ impl BasinClient { start: ReadStart, end: ReadEnd, encryption: Option<&EncryptionKey>, - stream_config: Option<&StreamConfig>, ) -> Result { let url = self.uri(format!("v1/streams/{}/records", urlencoding::encode(name))); let mut builder = self @@ -439,7 +432,6 @@ impl BasinClient { } let mut request = builder.build()?; set_encryption_header(&mut request, encryption); - set_stream_config_header(&mut request, stream_config); let response = self .request(request) .error_handler(read_response_error_handler) @@ -453,9 +445,7 @@ impl BasinClient { name: &StreamName, inputs: I, encryption: Option<&EncryptionKey>, - stream_config: Option<&StreamConfig>, frame_signal: Option, - reconnect: ReconnectAdvice, ) -> Result, ApiError> where I: Stream + Send + 'static, @@ -484,7 +474,6 @@ impl BasinClient { add_basin_header_if_required(request_builder, &self.config.endpoints, &self.name); let mut request = request_builder.build()?; set_encryption_header(&mut request, encryption); - set_stream_config_header(&mut request, stream_config); let (response, access_token) = self.client.init_streaming_authorized(request).await?; let response = match response.into_result().await { Ok(response) => response, @@ -494,15 +483,13 @@ impl BasinClient { return Err(error); } }; - let (mut bytes_stream, poison_handle) = response.into_stream(); + let mut bytes_stream = response.stream(); let auth_client = self.client.clone(); let mut buffer = BytesMut::new(); let mut decoder = FrameDecoder; Ok(Box::pin(try_stream! { - let mut advice_seen = false; - while let Some(chunk) = bytes_stream.next().await { let chunk = chunk?; buffer.extend_from_slice(&chunk); @@ -510,18 +497,10 @@ impl BasinClient { loop { match decoder.decode(&mut buffer) { Ok(Some(SessionMessage::Regular(msg))) => { - if !advice_seen && msg.reconnect_advised() { - advice_seen = true; - poison_handle.poison(); - reconnect.advise(); - } yield msg.try_into_proto()?; } Ok(Some(SessionMessage::Terminal(msg))) => { let error: ApiError = msg.into(); - if error.is_server_draining() { - poison_handle.poison(); - } auth_client.invalidate_access_token_if_rejected( &error, access_token.as_deref(), @@ -547,8 +526,6 @@ impl BasinClient { start: ReadStart, end: ReadEnd, encryption: Option<&EncryptionKey>, - stream_config: Option<&StreamConfig>, - reconnect: ReconnectAdvice, ) -> Result, ApiError> { let url = self.uri(format!("v1/streams/{}/records", urlencoding::encode(name))); @@ -563,7 +540,6 @@ impl BasinClient { add_basin_header_if_required(request_builder, &self.config.endpoints, &self.name); let mut request = request_builder.build()?; set_encryption_header(&mut request, encryption); - set_stream_config_header(&mut request, stream_config); let (response, access_token) = self.client.init_streaming_authorized(request).await?; let response = match response.into_result().await { Ok(response) => response, @@ -573,15 +549,13 @@ impl BasinClient { return Err(error); } }; - let (mut bytes_stream, poison_handle) = response.into_stream(); + let mut bytes_stream = response.stream(); let auth_client = self.client.clone(); let mut buffer = BytesMut::new(); let mut decoder = FrameDecoder; Ok(Box::pin(try_stream! { - let mut advice_seen = false; - while let Some(chunk) = bytes_stream.next().await { let chunk = chunk?; buffer.extend_from_slice(&chunk); @@ -589,18 +563,10 @@ impl BasinClient { loop { match decoder.decode(&mut buffer) { Ok(Some(SessionMessage::Regular(msg))) => { - if !advice_seen && msg.reconnect_advised() { - advice_seen = true; - poison_handle.poison(); - reconnect.advise(); - } yield msg.try_into_proto()?; } Ok(Some(SessionMessage::Terminal(msg))) => { let error: ApiError = msg.into(); - if error.is_server_draining() { - poison_handle.poison(); - } auth_client.invalidate_access_token_if_rejected( &error, access_token.as_deref(), @@ -667,13 +633,6 @@ pub(crate) enum ApiError { Compression(#[from] std::io::Error), #[error("append condition check failed")] AppendConditionFailed(AppendConditionFailed), - #[error( - "append may have taken effect in an earlier attempt; final attempt failed: {final_attempt_error}" - )] - IndefiniteFailure { - #[source] - final_attempt_error: Box, - }, #[error("read from an unwritten position")] ReadUnwritten(TailResponse), #[error("{1}")] @@ -685,50 +644,30 @@ impl ApiError { match self { Self::Server(status, err_resp) => server_error_is_retryable(*status, &err_resp.code), Self::Client(err) => err.is_retryable(), - Self::IndefiniteFailure { - final_attempt_error, - } => final_attempt_error.is_retryable(), #[cfg(feature = "_hidden")] Self::AccessTokenProvider(error) => error.is_retryable(), _ => false, } } - pub(crate) fn is_server_draining(&self) -> bool { - matches!( - self, - Self::Server(StatusCode::SERVICE_UNAVAILABLE, response) - if response.code == "server_draining" - ) - } - pub(crate) fn is_authentication_error(&self) -> bool { matches!( self, Self::Server(StatusCode::UNAUTHORIZED, response) if response.code == "authn" ) } -} -impl AppendRetryError for ApiError { - fn has_no_side_effects(&self) -> bool { + pub fn has_no_side_effects(&self) -> bool { match self { Self::Server(status, err_resp) => { server_error_has_no_side_effects(*status, &err_resp.code) } Self::Client(err) => err.has_no_side_effects(), - Self::AppendConditionFailed(_) | Self::MalformedAccessToken(_) => true, #[cfg(feature = "_hidden")] Self::AccessTokenProvider(_) => true, _ => false, } } - - fn into_indefinite_failure(self) -> Self { - Self::IndefiniteFailure { - final_attempt_error: Box::new(self), - } - } } impl From for ApiError { @@ -824,10 +763,7 @@ impl BaseClient { C: client::Connect + Clone + Send + Sync + 'static, { let access_token_mode = config.access_token.mode(); - let mut default_headers = config.default_headers.clone(); - // Authorization belongs to the configured token, including when a - // refreshable provider supplies it immediately before each attempt. - default_headers.remove(AUTHORIZATION); + let mut default_headers = HeaderMap::new(); #[cfg(feature = "_hidden")] let mut access_token_provider = None; match &config.access_token { @@ -998,14 +934,6 @@ fn set_encryption_header(request: &mut client::Request, encryption: Option<&Encr } } -fn set_stream_config_header(request: &mut client::Request, stream_config: Option<&StreamConfig>) { - if let Some(config) = stream_config { - request - .headers_mut() - .insert(STREAM_CONFIG_HEADER.clone(), config.to_header_value()); - } -} - pub fn retry_builder(config: &RetryConfig) -> RetryBackoffBuilder { RetryBackoffBuilder::default() .with_min_base_delay(config.min_base_delay) @@ -1054,7 +982,6 @@ impl<'a> RequestBuilder<'a> { let mut retry_backoff: Option = self .retry_enabled .then(|| self.client.retry_builder.build()); - let mut prior_uncertainty = false; loop { if let Some(ref signal) = self.frame_signal { @@ -1132,11 +1059,6 @@ impl<'a> RequestBuilder<'a> { num_retries_remaining = retry_backoff.as_ref().map(|b| b.remaining()).unwrap_or(0), "retrying request" ); - if self.append_retry_policy.is_some() - && err.attempt_may_have_side_effects(self.frame_signal.as_ref()) - { - prior_uncertainty = true; - } tokio::time::sleep(backoff).await; } else { debug!( @@ -1146,7 +1068,7 @@ impl<'a> RequestBuilder<'a> { retries_exhausted = retry_backoff.as_ref().is_none_or(|b| b.is_exhausted()), "not retrying request" ); - return Err(err.with_prior_uncertainty(prior_uncertainty)); + return Err(err); } } } @@ -1160,7 +1082,9 @@ fn is_safe_to_retry( ) -> bool { let policy_compliant = match policy { None | Some(AppendRetryPolicy::All) => true, - Some(AppendRetryPolicy::NoSideEffects) => !err.attempt_may_have_side_effects(frame_signal), + Some(AppendRetryPolicy::NoSideEffects) => { + !frame_signal.is_none_or(|s| s.is_signalled()) || err.has_no_side_effects() + } }; policy_compliant && (err.is_retryable() @@ -1246,20 +1170,14 @@ impl StreamingResult for StreamingResponse { } let status = self.status(); - let (bytes, poison_handle) = self.into_bytes_with_poison_handle().await?; + let bytes = self.into_bytes().await?; if status == StatusCode::RANGE_NOT_SATISFIABLE && let Ok(tail) = serde_json::from_slice::(&bytes) { return Err(ApiError::ReadUnwritten(tail)); } match serde_json::from_slice::(&bytes) { - Ok(response) => { - let error = ApiError::Server(status, response); - if error.is_server_draining() { - poison_handle.poison(); - } - Err(error) - } + Ok(response) => Err(ApiError::Server(status, response)), Err(error) => Err(ApiError::Client(ClientError::ResponseDecode(format!( "could not decode server error {status}: {error}; body: {}", String::from_utf8_lossy(&bytes), @@ -1306,11 +1224,11 @@ fn provision_result_from_parts( #[cfg(test)] mod tests { - use std::error::Error; #[cfg(feature = "_hidden")] - use std::sync::Mutex; - #[cfg(feature = "_hidden")] - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::{ + Mutex, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }; #[cfg(feature = "_hidden")] use async_trait::async_trait; @@ -1318,243 +1236,6 @@ mod tests { use hyper_util::client::legacy::connect::HttpConnector; use super::*; - use crate::error::AppendError; - - #[cfg(feature = "_hidden")] - #[derive(Default)] - struct HeaderCapture { - unary: Mutex>, - streaming: Mutex>, - } - - #[cfg(feature = "_hidden")] - #[async_trait] - impl client::RequestExecutor for HeaderCapture { - async fn execute_unary( - &self, - mut request: client::Request, - ) -> Result { - let mut headers = self.unary.lock().unwrap(); - headers.push(request.headers_mut().clone()); - // Exercise a real retry through RequestBuilder::send. - Ok(if headers.len() == 1 { - UnaryResponse::new_for_test( - StatusCode::SERVICE_UNAVAILABLE, - br#"{"code":"unavailable","message":"retry"}"#.to_vec(), - ) - } else { - UnaryResponse::new_for_test( - StatusCode::OK, - br#"{"basins":[],"streams":[],"has_more":false}"#.to_vec(), - ) - }) - } - - async fn init_streaming( - &self, - mut request: client::Request, - ) -> Result { - self.streaming - .lock() - .unwrap() - .push(request.headers_mut().clone()); - // Capturing initiation is sufficient: do not construct a response body. - Err(client::HttpError::Timeout) - } - } - - #[cfg(feature = "_hidden")] - #[tokio::test] - async fn default_headers_reach_account_basin_streaming_and_retry_requests() { - let mut session = HeaderValue::from_static("session-1"); - session.set_sensitive(true); - let headers = HeaderMap::from_iter([ - ( - http::header::HeaderName::from_static("x-origin-session"), - session, - ), - ( - AUTHORIZATION, - HeaderValue::from_static("Bearer wrong-token"), - ), - ( - http::header::USER_AGENT, - HeaderValue::from_static("wrong-agent"), - ), - ( - http::header::ACCEPT_ENCODING, - HeaderValue::from_static("wrong-encoding"), - ), - ( - http::header::HeaderName::from_static(S2_BASIN), - HeaderValue::from_static("wrong-basin"), - ), - ]); - let config = S2Config::new("actual-token") - .with_endpoints(S2Endpoints::for_endpoint("http://example.test").unwrap()) - .with_compression(Compression::Gzip) - .with_default_headers(headers) - .unwrap(); - let executor = Arc::new(HeaderCapture::default()); - let mut base = BaseClient::init_with_connector(&config, HttpConnector::new()).unwrap(); - base.client = executor.clone(); - base.retry_builder = RetryBackoffBuilder::default() - .with_min_base_delay(Duration::ZERO) - .with_max_base_delay(Duration::ZERO) - .with_max_retries(1); - let account = AccountClient::init(config.clone(), base); - account - .list_basins(ListBasinsRequest { - prefix: None, - start_after: None, - limit: None, - }) - .await - .unwrap(); - let basin = account.basin_client("test-basin".parse().unwrap()); - basin - .list_streams(ListStreamsRequest { - prefix: None, - start_after: None, - limit: None, - }) - .await - .unwrap(); - - let stream = "test-stream".parse().unwrap(); - assert!( - basin - .read_session( - &stream, - ReadStart { - seq_num: None, - timestamp: None, - tail_offset: None, - clamp: None - }, - ReadEnd { - count: None, - bytes: None, - until: None, - wait: None - }, - None, - None, - ReconnectAdvice::default(), - ) - .await - .is_err() - ); - assert!( - basin - .append_session( - &stream, - futures_util::stream::empty(), - None, - None, - None, - ReconnectAdvice::default(), - ) - .await - .is_err() - ); - - let unary = executor.unary.lock().unwrap(); - let streaming = executor.streaming.lock().unwrap(); - assert_eq!(unary.len(), 3); // account, account retry, basin - assert_eq!(streaming.len(), 2); // read and append - for headers in unary.iter().chain(streaming.iter()) { - assert_eq!(headers["x-origin-session"], "session-1"); - assert!(headers["x-origin-session"].is_sensitive()); - assert_eq!(headers[AUTHORIZATION], "Bearer actual-token"); - assert!(headers[AUTHORIZATION].is_sensitive()); - assert_eq!(headers[http::header::USER_AGENT], config.user_agent); - assert_eq!(headers[http::header::ACCEPT_ENCODING], "gzip"); - assert!(!headers.contains_key(http::header::CONTENT_ENCODING)); - } - assert_eq!(unary[2][S2_BASIN], "test-basin"); - for headers in streaming.iter() { - assert_eq!(headers[S2_BASIN], "test-basin"); - assert_eq!(headers[CONTENT_TYPE], CONTENT_TYPE_S2S); - } - } - - #[cfg(feature = "_hidden")] - #[rstest::rstest] - #[case::none(Compression::None, None, "gzip, zstd")] - #[case::gzip(Compression::Gzip, Some("gzip"), "gzip")] - #[case::zstd(Compression::Zstd, Some("zstd"), "zstd")] - #[tokio::test] - async fn default_accept_encoding_respects_configured_compression( - #[case] compression: Compression, - #[case] content_encoding: Option<&str>, - #[case] accept_encoding: &str, - ) { - let config = S2Config::new("token") - .with_compression(compression) - .with_default_headers(HeaderMap::from_iter([( - http::header::ACCEPT_ENCODING, - HeaderValue::from_static("gzip, zstd"), - )])) - .unwrap(); - let client = BaseClient::init_with_connector(&config, HttpConnector::new()).unwrap(); - let mut request = client - .post("http://example.test/v1/basins".parse().unwrap()) - .json(&serde_json::json!({"name": "test-basin"})) - .build() - .unwrap() - .compress() - .await - .unwrap(); - let headers = request.headers_mut(); - assert_eq!( - headers - .get(http::header::CONTENT_ENCODING) - .map(|value| value.to_str().unwrap()), - content_encoding - ); - assert_eq!(headers[http::header::ACCEPT_ENCODING], accept_encoding); - } - - #[cfg(feature = "_hidden")] - #[tokio::test] - async fn default_headers_are_replaced_and_isolated_between_clients() { - let first_headers = HeaderMap::from_iter([ - ( - http::header::HeaderName::from_static("x-origin-session"), - HeaderValue::from_static("first"), - ), - ( - http::header::HeaderName::from_static("x-first-only"), - HeaderValue::from_static("present"), - ), - ]); - let first = S2Config::new("token") - .with_default_headers(first_headers) - .unwrap(); - let second = first - .clone() - .with_default_headers(HeaderMap::from_iter([( - http::header::HeaderName::from_static("x-origin-session"), - HeaderValue::from_static("second"), - )])) - .unwrap(); - for (config, session) in [(&first, "first"), (&second, "second")] { - let client = BaseClient::init_with_connector(config, HttpConnector::new()).unwrap(); - let mut request = client - .get("http://example.test".parse().unwrap()) - .build() - .unwrap(); - assert_eq!(request.headers_mut()["x-origin-session"], session); - assert_eq!( - request.headers_mut().contains_key("x-first-only"), - session == "first" - ); - } - let cleared = second.with_default_headers(HeaderMap::new()).unwrap(); - assert!(cleared.default_headers.is_empty()); - assert!(S2Config::new("token").default_headers.is_empty()); - } #[cfg(feature = "_hidden")] #[derive(Debug)] @@ -1641,78 +1322,6 @@ mod tests { } } - #[rstest::rstest] - #[case(StatusCode::FORBIDDEN, "permission_denied", false, false)] - #[case(StatusCode::FORBIDDEN, "permission_denied", true, true)] - #[case(StatusCode::SERVICE_UNAVAILABLE, "unavailable", false, false)] - #[case(StatusCode::SERVICE_UNAVAILABLE, "unavailable", true, false)] - #[case(StatusCode::TOO_MANY_REQUESTS, "rate_limited", true, true)] - #[test] - fn unary_append_failure_preserves_uncertainty_and_final_error( - #[case] status: StatusCode, - #[case] code: &str, - #[case] prior_uncertainty: bool, - #[case] wrapped: bool, - ) { - let error = - AppendError::from(server_error(status, code).with_prior_uncertainty(prior_uncertainty)); - assert_eq!( - matches!(error, AppendError::IndefiniteFailure { .. }), - wrapped - ); - let server = error.request_error().unwrap().server_error().unwrap(); - assert_eq!((server.status, server.code.as_str()), (status, code)); - assert_eq!(server.message, "test"); - assert_eq!(error.is_retryable(), server.is_retryable()); - assert_eq!( - error.has_no_side_effects(), - !wrapped && server.has_no_side_effects() - ); - if wrapped { - let latest = error - .source() - .unwrap() - .downcast_ref::>() - .unwrap(); - assert!(latest.has_no_side_effects()); - assert_eq!( - latest.request_error().unwrap().server_error().unwrap().code, - code - ); - assert_eq!( - error.to_string(), - format!( - "append may have taken effect in an earlier attempt; final attempt failed: {latest}" - ) - ); - } - } - - #[rstest::rstest] - #[case::condition_failed( - ApiError::AppendConditionFailed(AppendConditionFailed::SeqNumMismatch(42)), - "sequence number mismatch, expected: 42" - )] - #[case::malformed_access_token( - ApiError::MalformedAccessToken("invalid header".to_owned()), - "malformed access token: invalid header" - )] - #[test] - fn unary_append_wraps_definite_terminal_errors(#[case] error: ApiError, #[case] message: &str) { - assert!(error.has_no_side_effects()); - let error = AppendError::from(error.with_prior_uncertainty(true)); - assert!(matches!(error, AppendError::IndefiniteFailure { .. })); - assert!(!error.has_no_side_effects()); - assert!(!error.is_retryable()); - let latest = error - .source() - .unwrap() - .downcast_ref::>() - .unwrap(); - assert!(latest.has_no_side_effects()); - assert_eq!(latest.to_string(), message); - } - fn server_error(status: StatusCode, code: &str) -> ApiError { ApiError::Server( status, @@ -1789,15 +1398,9 @@ mod tests { #[cfg(feature = "_hidden")] #[tokio::test] async fn dynamic_access_token_is_loaded_for_each_attempt_and_marked_sensitive() { - let config = S2Config::new("unused") - .with_default_headers(HeaderMap::from_iter([( - AUTHORIZATION, - HeaderValue::from_static("Bearer wrong-token"), - )])) - .unwrap() - .with_access_token_provider(RotatingTokenProvider { - generation: AtomicUsize::new(1), - }); + let config = S2Config::new("unused").with_access_token_provider(RotatingTokenProvider { + generation: AtomicUsize::new(1), + }); let client = BaseClient::init_with_connector(&config, HttpConnector::new()).unwrap(); let uri = "http://example.test/v1/basins".parse().unwrap(); let mut request = client.get(uri).build().unwrap(); diff --git a/sdk/src/batching.rs b/sdk/src/batching.rs index eac2c5ab..00086f1f 100644 --- a/sdk/src/batching.rs +++ b/sdk/src/batching.rs @@ -174,7 +174,6 @@ impl Stream for AppendInputs { records: batch, match_seq_num, fencing_token: self.fencing_token.clone(), - stream_config: None, }))) } Poll::Ready(Some(Err(err))) => Poll::Ready(Some(Err(err))), diff --git a/sdk/src/client.rs b/sdk/src/client.rs index abf927ab..b146e975 100644 --- a/sdk/src/client.rs +++ b/sdk/src/client.rs @@ -2,8 +2,8 @@ use std::{ collections::HashMap, convert::Infallible, sync::{ - Arc, Mutex, RwLock as StdRwLock, - atomic::{AtomicU64, AtomicUsize, Ordering}, + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, }, time::{Duration, Instant}, }; @@ -29,7 +29,7 @@ use hyper_rustls::{HttpsConnector, HttpsConnectorBuilder}; pub use hyper_util::client::legacy::connect::Connect; use hyper_util::{ client::legacy::{Client as HyperClient, connect::HttpConnector}, - rt::{TokioExecutor, TokioTimer}, + rt::TokioExecutor, }; use serde::{Serialize, de::DeserializeOwned}; use tokio::{ @@ -304,8 +304,9 @@ impl RequestBuilder { } pub fn headers(mut self, headers: &HeaderMap) -> Self { - // An owned HeaderMap replaces each key's existing values while preserving duplicates. - self.headers.extend(headers.clone()); + for (key, value) in headers { + self.headers.insert(key.clone(), value.clone()); + } self } @@ -368,63 +369,20 @@ impl UnaryResponse { } } -/// Identifies a pooled connection within its host pool, so poisoning drops -/// just the connection reconnect advice arrived on. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct ConnectionId(u64); - -impl ConnectionId { - fn next() -> Self { - static NEXT: AtomicU64 = AtomicU64::new(0); - Self(NEXT.fetch_add(1, Ordering::Relaxed)) - } -} - -/// Grants its holder the ability to poison the pooled connection a streaming -/// response was served on, dropping it from the pool so no new request reuses -/// it. Requests already in flight keep the connection. -/// -/// Poisoning is idempotent: the connection is identified by its -/// [`ConnectionId`], so poisoning it again — including from another session -/// sharing the connection — is a no-op. -pub struct PoisonHandle { - poison: Box, -} - -impl PoisonHandle { - fn new(poison: impl Fn() + Send + Sync + 'static) -> Self { - Self { - poison: Box::new(poison), - } - } - - pub(crate) fn poison(&self) { - (self.poison)(); - } -} - pub struct StreamingResponse { status: StatusCode, headers: HeaderMap, body: Incoming, permit: RequestPermit, - poison_handle: PoisonHandle, } impl StreamingResponse { - fn new( - status: StatusCode, - headers: HeaderMap, - body: Incoming, - permit: RequestPermit, - poison_handle: PoisonHandle, - ) -> Self { + fn new(status: StatusCode, headers: HeaderMap, body: Incoming, permit: RequestPermit) -> Self { Self { status, headers, body, permit, - poison_handle, } } @@ -432,37 +390,20 @@ impl StreamingResponse { self.status } - pub(crate) async fn into_bytes_with_poison_handle( - self, - ) -> Result<(Bytes, PoisonHandle), HttpError> { - let Self { - headers, - body, - permit, - poison_handle, - .. - } = self; - let bytes = body.collect().await?.to_bytes(); - let bytes = decompress_body(&headers, bytes).await?; - drop(permit); - Ok((bytes, poison_handle)) + pub async fn into_bytes(self) -> Result { + let bytes = self.body.collect().await?.to_bytes(); + decompress_body(&self.headers, bytes).await } - pub fn into_stream(self) -> (impl Stream>, PoisonHandle) { - let Self { - body, - permit, - poison_handle, - .. - } = self; - let stream = http_body_util::BodyStream::new(body).filter_map(move |result| { + pub fn stream(self) -> impl Stream> { + let permit = self.permit; + http_body_util::BodyStream::new(self.body).filter_map(move |result| { let _ = &permit; std::future::ready(match result { Ok(frame) => frame.into_data().ok().map(Ok), Err(e) => Some(Err(HttpError::Receive(e))), }) - }); - (stream, poison_handle) + }) } } @@ -573,7 +514,11 @@ fn build_http_request( let mut builder = http::Request::builder().method(method).uri(uri.clone()); if let Some(req_headers) = builder.headers_mut() { - *req_headers = headers; + for (key, value) in headers { + if let Some(key) = key { + req_headers.insert(key, value); + } + } if let Some(encoding) = content_encoding { req_headers.insert(CONTENT_ENCODING, encoding); } @@ -632,7 +577,6 @@ async fn init_streaming_with( client: &HyperClient, request: Request, permit: RequestPermit, - poison_handle: PoisonHandle, ) -> Result where C: Connect + Clone + Send + Sync + 'static, @@ -656,7 +600,6 @@ where parts.headers, body, permit, - poison_handle, )) }; @@ -786,7 +729,6 @@ impl Drop for RequestPermit { } struct PooledClient { - id: ConnectionId, client: Arc>, active_requests: Arc, idle_since: Arc>>, @@ -795,7 +737,6 @@ struct PooledClient { impl PooledClient { fn new(client: HyperClient) -> Self { Self { - id: ConnectionId::next(), client: Arc::new(client), active_requests: Arc::new(AtomicUsize::new(0)), idle_since: Arc::new(Mutex::new(Some(Instant::now()))), @@ -827,7 +768,7 @@ impl PooledClient { } struct HostPool { - clients: StdRwLock>>, + clients: RwLock>>, connector: C, } @@ -837,34 +778,29 @@ where { fn new(connector: C) -> Self { Self { - clients: StdRwLock::new(Vec::new()), + clients: RwLock::new(Vec::new()), connector, } } fn create_client(&self) -> PooledClient { - let client = HyperClient::builder(TokioExecutor::new()) - .timer(TokioTimer::new()) - .http2_only(true) - .http2_keep_alive_interval(Duration::from_secs(20)) - .http2_keep_alive_timeout(Duration::from_secs(10)) - .build(self.connector.clone()); + let client = HyperClient::builder(TokioExecutor::new()).build(self.connector.clone()); PooledClient::new(client) } - fn checkout(&self) -> (Arc>, RequestPermit, ConnectionId) { + async fn checkout(&self) -> (Arc>, RequestPermit) { { - let clients = self.clients.read().unwrap(); + let clients = self.clients.read().await; for pooled in clients.iter() { if let Some(permit) = pooled.request_permit() { - return (pooled.client.clone(), permit, pooled.id); + return (pooled.client.clone(), permit); } } } - let mut clients = self.clients.write().unwrap(); + let mut clients = self.clients.write().await; for pooled in clients.iter() { if let Some(permit) = pooled.request_permit() { - return (pooled.client.clone(), permit, pooled.id); + return (pooled.client.clone(), permit); } } let new_client = self.create_client(); @@ -872,27 +808,14 @@ where .request_permit() .expect("new client must have a permit"); let client = new_client.client.clone(); - let id = new_client.id; clients.push(new_client); - (client, permit, id) + (client, permit) } - /// Drop the pooled client identified by `id` so no new request reuses it. - /// Clients pinned to servers that are not going away stay pooled, requests - /// already in flight keep their connection, and poisoning the same - /// connection again is a no-op. - fn poison(&self, host: &str, id: ConnectionId) { - let mut clients = self.clients.write().unwrap(); - let pooled = clients.len(); - clients.retain(|pooled| pooled.id != id); - let removed = pooled - clients.len(); - tracing::debug!(host, connection = ?id, removed, "poisoned pooled connections"); - } - - fn reap_idle_clients(&self) { + async fn reap_idle_clients(&self) { self.clients .write() - .unwrap() + .await .retain(|pooled| !pooled.should_reap(IDLE_TIMEOUT)); } @@ -949,11 +872,8 @@ where .clone() } - async fn checkout( - &self, - host: &str, - ) -> (Arc>, RequestPermit, ConnectionId) { - self.get_or_create_host_pool(host).await.checkout() + async fn checkout(&self, host: &str) -> (Arc>, RequestPermit) { + self.get_or_create_host_pool(host).await.checkout().await } } @@ -966,7 +886,7 @@ async fn reap_idle_clients( }; for pool in &pools { - pool.reap_idle_clients(); + pool.reap_idle_clients().await; } hosts.write().await.retain(|_, pool| !pool.is_empty()); @@ -978,23 +898,13 @@ where C: Connect + Clone + Send + Sync + 'static, { async fn execute_unary(&self, request: Request) -> Result { - let (client, _permit, _) = self.checkout(request.authority()).await; + let (client, _permit) = self.checkout(request.authority()).await; execute_unary_with(&client, request).await } async fn init_streaming(&self, request: Request) -> Result { - let host = request.authority().to_owned(); - let pool = self.get_or_create_host_pool(&host).await; - let (client, permit, id) = pool.checkout(); - // Weak: the handle can outlive the pool (a session holds it for the - // connection's lifetime) and must not keep a reaped pool alive. - let weak = Arc::downgrade(&pool); - let poison_handle = PoisonHandle::new(move || { - if let Some(pool) = weak.upgrade() { - pool.poison(&host, id); - } - }); - init_streaming_with(&client, request, permit, poison_handle).await + let (client, permit) = self.checkout(request.authority()).await; + init_streaming_with(&client, request, permit).await } } @@ -1011,37 +921,6 @@ mod tests { Pool::new(HttpConnector::new()) } - #[test] - fn default_headers_preserve_multiple_values_and_sensitive_flags_on_wire_request() { - let mut headers = HeaderMap::new(); - headers.append("x-tag", HeaderValue::from_static("first")); - let mut second = HeaderValue::from_static("second"); - second.set_sensitive(true); - headers.append("x-tag", second); - headers.insert(CONTENT_ENCODING, HeaderValue::from_static("wrong-encoding")); - - let request = RequestBuilder::get("http://example.test".parse().unwrap()) - .header("x-tag", "replaced") - .headers(&headers) - .build() - .unwrap(); - let cloned = request.try_clone().unwrap(); - let http = build_http_request( - cloned.method, - &cloned.uri, - cloned.headers, - cloned.body.into_http_body(), - Some(HeaderValue::from_static("gzip")), - ) - .unwrap(); - let values = http.headers().get_all("x-tag").iter().collect::>(); - assert_eq!(values.len(), 2); - assert_eq!(values[0], "first"); - assert_eq!(values[1], "second"); - assert!(values[1].is_sensitive()); - assert_eq!(http.headers()[CONTENT_ENCODING], "gzip"); - } - #[test] fn uri_with_path_percent_encoded_segment() { let base: Uri = "https://example.com".parse().unwrap(); @@ -1076,7 +955,7 @@ mod tests { async fn host_client_count(pool: &Pool, host: &str) -> usize { let hosts = pool.hosts.read().await; match hosts.get(host) { - Some(pool) => pool.clients.read().unwrap().len(), + Some(pool) => pool.clients.read().await.len(), None => 0, } } @@ -1119,7 +998,7 @@ mod tests { let pool = test_pool(); let mut permits = Vec::new(); for _ in 0..MAX_CONCURRENT_REQUESTS_PER_CLIENT { - let (_client, permit, _) = pool.checkout(TEST_HOST).await; + let (_client, permit) = pool.checkout(TEST_HOST).await; permits.push(permit); } assert_eq!(host_client_count(&pool, TEST_HOST).await, 1); @@ -1130,12 +1009,12 @@ mod tests { let pool = test_pool(); let mut permits = Vec::new(); for _ in 0..MAX_CONCURRENT_REQUESTS_PER_CLIENT { - let (_client, permit, _) = pool.checkout(TEST_HOST).await; + let (_client, permit) = pool.checkout(TEST_HOST).await; permits.push(permit); } assert_eq!(host_client_count(&pool, TEST_HOST).await, 1); - let (_client, permit, _) = pool.checkout(TEST_HOST).await; + let (_client, permit) = pool.checkout(TEST_HOST).await; permits.push(permit); assert_eq!(host_client_count(&pool, TEST_HOST).await, 2); } @@ -1145,12 +1024,12 @@ mod tests { let pool = test_pool(); let mut permits = Vec::new(); for _ in 0..MAX_CONCURRENT_REQUESTS_PER_CLIENT { - let (_client, permit, _) = pool.checkout(TEST_HOST).await; + let (_client, permit) = pool.checkout(TEST_HOST).await; permits.push(permit); } permits.pop(); - let (_client, permit, _) = pool.checkout(TEST_HOST).await; + let (_client, permit) = pool.checkout(TEST_HOST).await; permits.push(permit); assert_eq!(host_client_count(&pool, TEST_HOST).await, 1); } @@ -1160,10 +1039,10 @@ mod tests { let pool = test_pool(); let mut permits = Vec::new(); for _ in 0..MAX_CONCURRENT_REQUESTS_PER_CLIENT { - let (_client, permit, _) = pool.checkout(TEST_HOST).await; + let (_client, permit) = pool.checkout(TEST_HOST).await; permits.push(permit); } - let (_client, permit, _) = pool.checkout(TEST_HOST).await; + let (_client, permit) = pool.checkout(TEST_HOST).await; permits.push(permit); assert_eq!(host_client_count(&pool, TEST_HOST).await, 2); @@ -1171,7 +1050,7 @@ mod tests { { let hosts = pool.hosts.read().await; let pool = hosts.get(TEST_HOST).unwrap(); - let clients = pool.clients.read().unwrap(); + let clients = pool.clients.read().await; for pooled in clients.iter() { *pooled.idle_since.lock().unwrap() = Some(Instant::now() - IDLE_TIMEOUT - Duration::from_secs(1)); @@ -1191,12 +1070,12 @@ mod tests { let mut permits_a = Vec::new(); for _ in 0..MAX_CONCURRENT_REQUESTS_PER_CLIENT { - let (_client, permit, _) = pool.checkout(host_a).await; + let (_client, permit) = pool.checkout(host_a).await; permits_a.push(permit); } assert_eq!(host_client_count(&pool, host_a).await, 1); - let (_client, permit_b, _) = pool.checkout(host_b).await; + let (_client, permit_b) = pool.checkout(host_b).await; assert_eq!(host_client_count(&pool, host_b).await, 1); assert_eq!(host_client_count(&pool, host_a).await, 1); @@ -1208,15 +1087,15 @@ mod tests { async fn reaper_removes_empty_host_entries() { let pool = test_pool(); - let (_client, permit_a, _) = pool.checkout("host-a:443").await; - let (_client, permit_b, _) = pool.checkout("host-b:443").await; + let (_client, permit_a) = pool.checkout("host-a:443").await; + let (_client, permit_b) = pool.checkout("host-b:443").await; assert_eq!(pool.hosts.read().await.len(), 2); drop(permit_a); { let hosts = pool.hosts.read().await; let pool_a = hosts.get("host-a:443").unwrap(); - let clients = pool_a.clients.read().unwrap(); + let clients = pool_a.clients.read().await; for pooled in clients.iter() { *pooled.idle_since.lock().unwrap() = Some(Instant::now() - IDLE_TIMEOUT - Duration::from_secs(1)); diff --git a/sdk/src/error.rs b/sdk/src/error.rs index bdeb5e53..a99cc1c8 100644 --- a/sdk/src/error.rs +++ b/sdk/src/error.rs @@ -122,22 +122,10 @@ impl From for ClientError { fn classify_hyper_source(err: &client::HttpError, err_msg: &str) -> Option { let hyper_err = source_err::(err)?; let err_msg = format!("{hyper_err} -> {err_msg}"); - if hyper_err.is_timeout() { - // The h2 keep-alive timing out fails requests sent on the dead connection. - Some(ClientError::Timeout) - } else if hyper_err.is_incomplete_message() || hyper_err.is_closed() { - // `is_closed` covers a request dispatched onto a pooled connection - // that the server had already shut down. + if hyper_err.is_incomplete_message() { Some(ClientError::ConnectionClosedEarly(err_msg)) } else if hyper_err.is_canceled() { Some(ClientError::RequestCanceled(err_msg)) - } else if source_err::(err).is_some_and(|e| { - e.is_io() || e.is_go_away() || e.reason() == Some(h2::Reason::REFUSED_STREAM) - }) { - // An I/O failure ends streaming bodies without tripping any hyper marker above. - // A remote GOAWAY ends streams dispatched onto a connection the server is - // gracefully shutting down. - Some(ClientError::ConnectionClosedEarly(err_msg)) } else { None } @@ -148,8 +136,6 @@ fn classify_io_source(err: &client::HttpError, err_msg: &str) -> Option {err_msg}"); Some(match io_err.kind() { std::io::ErrorKind::UnexpectedEof => ClientError::UnexpectedEof(err_msg), - // h2 surfaces a stream cut short by connection shutdown as a broken pipe. - std::io::ErrorKind::BrokenPipe => ClientError::ConnectionClosedEarly(err_msg), std::io::ErrorKind::ConnectionReset => ClientError::ConnectionReset(err_msg), std::io::ErrorKind::ConnectionAborted => ClientError::ConnectionAborted(err_msg), std::io::ErrorKind::ConnectionRefused => ClientError::ConnectionRefused(err_msg), @@ -251,15 +237,6 @@ impl RequestError { if error.status == StatusCode::UNAUTHORIZED && error.code == "authn" ) } - - pub(crate) fn is_server_draining(&self) -> bool { - matches!( - self, - Self::Server(error) - if error.status == StatusCode::SERVICE_UNAVAILABLE - && error.code == "server_draining" - ) - } } impl From for RequestError { @@ -332,28 +309,12 @@ pub enum AppendError { /// The append condition did not match. #[error(transparent)] ConditionFailed(#[from] AppendConditionFailed), - /// The final attempt failed definitively, but an earlier attempt may have taken effect, - /// so the entire append operation is indeterminate. - #[error( - "append may have taken effect in an earlier attempt; final attempt failed: {final_attempt_error}" - )] - IndefiniteFailure { - /// The definite error returned by the final attempt. - #[source] - final_attempt_error: Box, - }, } impl AppendError { /// Whether retrying the operation is safe or sensible. pub fn is_retryable(&self) -> bool { - match self { - Self::Request(error) => error.is_retryable(), - Self::ConditionFailed(_) => false, - Self::IndefiniteFailure { - final_attempt_error, - } => final_attempt_error.is_retryable(), - } + matches!(self, Self::Request(error) if error.is_retryable()) } /// Whether retrying the operation cannot duplicate a mutation. @@ -361,7 +322,6 @@ impl AppendError { match self { Self::Request(error) => error.has_no_side_effects(), Self::ConditionFailed(_) => true, - Self::IndefiniteFailure { .. } => false, } } @@ -370,9 +330,6 @@ impl AppendError { match self { Self::Request(error) => Some(error), Self::ConditionFailed(_) => None, - Self::IndefiniteFailure { - final_attempt_error, - } => final_attempt_error.request_error(), } } } @@ -381,11 +338,6 @@ impl From for AppendError { fn from(error: ApiError) -> Self { match error { ApiError::AppendConditionFailed(condition) => Self::ConditionFailed(condition.into()), - ApiError::IndefiniteFailure { - final_attempt_error, - } => Self::IndefiniteFailure { - final_attempt_error: Box::new((*final_attempt_error).into()), - }, other => Self::Request(other.into()), } } diff --git a/sdk/src/lib.rs b/sdk/src/lib.rs index 5699fea9..ded5890b 100644 --- a/sdk/src/lib.rs +++ b/sdk/src/lib.rs @@ -92,7 +92,6 @@ issue. mod api; mod client; mod frame_signal; -mod reconnect; mod session; pub mod batching; diff --git a/sdk/src/ops.rs b/sdk/src/ops.rs index db339709..be287bec 100644 --- a/sdk/src/ops.rs +++ b/sdk/src/ops.rs @@ -4,9 +4,7 @@ use crate::{ api::{AccountClient, BaseClient, BasinClient}, error::{AppendError, ReadError, RequestError}, producer::{Producer, ProducerConfig}, - session::{ - self, AppendSession, AppendSessionConfig, ReadSession, ReadSessionError, StreamHeaders, - }, + session::{self, AppendSession, AppendSessionConfig, ReadSession, ReadSessionError}, types::{ AccessTokenId, AccessTokenInfo, AppendAck, AppendInput, BasinConfig, BasinInfo, BasinName, CreateBasinInput, CreateStreamInput, DeleteBasinInput, DeleteStreamInput, EncryptionKey, @@ -439,13 +437,6 @@ impl S2Stream { } } - fn headers(&self, stream_config: Option<&StreamConfig>) -> StreamHeaders { - StreamHeaders { - encryption: self.encryption.clone(), - stream_config: stream_config.cloned().map(Into::into), - } - } - /// Check tail position. pub async fn check_tail(&self) -> Result { let response = self.client.check_tail(&self.name).await?; @@ -453,18 +444,13 @@ impl S2Stream { } /// Append records. - pub async fn append(&self, mut input: AppendInput) -> Result { - let stream_config = input - .stream_config - .take() - .map(s2_api::v1::config::StreamConfig::from); + pub async fn append(&self, input: AppendInput) -> Result { let ack = self .client .append( &self.name, input.into(), self.encryption.as_ref(), - stream_config.as_ref(), self.client.config.retry.append_retry_policy, ) .await?; @@ -473,9 +459,6 @@ impl S2Stream { /// Read records. pub async fn read(&self, input: ReadInput) -> Result { - let stream_config = input - .stream_config - .map(s2_api::v1::config::StreamConfig::from); let batch = self .client .read( @@ -483,7 +466,6 @@ impl S2Stream { input.start.into(), input.stop.into(), self.encryption.as_ref(), - stream_config.as_ref(), ) .await?; let mut batch = ReadBatch::from_api(batch); @@ -498,7 +480,7 @@ impl S2Stream { AppendSession::new( self.client.clone(), self.name.clone(), - self.headers(config.stream_config()), + self.encryption.clone(), config, ) } @@ -508,7 +490,7 @@ impl S2Stream { Producer::new( self.client.clone(), self.name.clone(), - self.headers(config.stream_config()), + self.encryption.clone(), config, ) } @@ -522,7 +504,7 @@ impl S2Stream { session::read_session( self.client.clone(), self.name.clone(), - self.headers(input.stream_config.as_ref()), + self.encryption.clone(), input, config, ) diff --git a/sdk/src/producer.rs b/sdk/src/producer.rs index 293f774a..632c31b5 100644 --- a/sdk/src/producer.rs +++ b/sdk/src/producer.rs @@ -20,11 +20,9 @@ use crate::{ api::BasinClient, batching::{AppendInputs, AppendRecordBatches, BatchingConfig}, error::ProducerError, - session::{ - AppendPermit, AppendPermits, AppendSessionInternal, BatchSubmitTicket, StreamHeaders, - }, + session::{AppendPermit, AppendPermits, AppendSessionInternal, BatchSubmitTicket}, types::{ - AppendAck, AppendRecord, FencingToken, MeteredBytes, ONE_MIB, StreamConfig, StreamName, + AppendAck, AppendRecord, EncryptionKey, FencingToken, MeteredBytes, ONE_MIB, StreamName, ValidationError, }, }; @@ -77,7 +75,6 @@ pub struct ProducerConfig { batching: BatchingConfig, fencing_token: Option, match_seq_num: Option, - stream_config: Option, } impl Default for ProducerConfig { @@ -87,7 +84,6 @@ impl Default for ProducerConfig { batching: BatchingConfig::default(), fencing_token: None, match_seq_num: None, - stream_config: None, } } } @@ -141,23 +137,6 @@ impl ProducerConfig { ..self } } - - /// Set the stream configuration to apply if the stream is created on append. - /// - /// Unset fields inherit the basin's default stream configuration. Ignored if the stream - /// already exists. - /// - /// Defaults to `None`. - pub fn with_stream_config(self, stream_config: StreamConfig) -> Self { - Self { - stream_config: Some(stream_config), - ..self - } - } - - pub(crate) fn stream_config(&self) -> Option<&StreamConfig> { - self.stream_config.as_ref() - } } /// High-level interface for submitting individual [`AppendRecord`]s. @@ -175,12 +154,12 @@ impl Producer { pub(crate) fn new( client: BasinClient, stream: StreamName, - headers: StreamHeaders, + encryption: Option, config: ProducerConfig, ) -> Self { let (cmd_tx, cmd_rx) = mpsc::channel::(RECORD_BATCH_MAX.count); let permits = AppendPermits::new(None, config.max_unacked_bytes); - let session = AppendSessionInternal::new(client, stream, headers); + let session = AppendSessionInternal::new(client, stream, encryption); let terminal_err = Arc::new(OnceLock::new()); let _handle = AbortOnDropHandle::new(tokio::spawn(Self::run( session, diff --git a/sdk/src/reconnect.rs b/sdk/src/reconnect.rs deleted file mode 100644 index 9df29362..00000000 --- a/sdk/src/reconnect.rs +++ /dev/null @@ -1,62 +0,0 @@ -use std::{ - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - time::Duration, -}; - -use tokio::time::Instant; - -/// Advised reconnects to attempt before staying on. -pub(crate) const MAX_ADVISED_RECONNECTS: usize = 1; - -/// Gap after which the attempt count resets. -pub(crate) const ADVISED_RECONNECT_IDLE: Duration = Duration::from_secs(60); - -/// Advised reconnects attempted lately. -#[derive(Clone, Copy, Default)] -pub(crate) struct AdvisedReconnects { - count: usize, - last: Option, -} - -impl AdvisedReconnects { - pub(crate) fn record(&mut self) { - if !self.is_recent() { - self.count = 0; - } - self.last = Some(Instant::now()); - self.count += 1; - } - - /// Whether to act on advice, or stay until the server ends the connection. - pub(crate) fn should_reconnect(&self) -> bool { - !self.is_recent() || self.count < MAX_ADVISED_RECONNECTS - } - - pub(crate) fn count(&self) -> usize { - self.count - } - - fn is_recent(&self) -> bool { - self.last - .is_some_and(|at| at.elapsed() <= ADVISED_RECONNECT_IDLE) - } -} - -/// An atomic flag tracking whether the server has advised reconnecting on this -/// connection. Set by the response decoder when a frame carries the -/// reconnect-advised bit, checked by the session loops. -#[derive(Clone, Default)] -pub(crate) struct ReconnectAdvice(Arc); - -impl ReconnectAdvice { - pub(crate) fn is_advised(&self) -> bool { - self.0.load(Ordering::Acquire) - } - - pub(crate) fn advise(&self) { - self.0.store(true, Ordering::Release); - } -} diff --git a/sdk/src/retry.rs b/sdk/src/retry.rs index 8eddd738..ac7bb370 100644 --- a/sdk/src/retry.rs +++ b/sdk/src/retry.rs @@ -2,33 +2,6 @@ use std::time::Duration; use rand::{RngExt, rng}; -use crate::frame_signal::FrameSignal; - -pub(crate) trait AppendRetryError: Sized { - /// Whether the failure represented by this error guarantees that no mutation occurred. - fn has_no_side_effects(&self) -> bool; - - /// Mark the whole append as indefinite, retaining this final attempt's definite error. - /// The caller must establish that an earlier attempt may have taken effect. - fn into_indefinite_failure(self) -> Self; - - /// Wrap a definite final error if an earlier attempt may have taken effect. - /// Already indefinite errors are returned unchanged. - fn with_prior_uncertainty(self, prior_uncertainty: bool) -> Self { - if prior_uncertainty && self.has_no_side_effects() { - self.into_indefinite_failure() - } else { - self - } - } - - /// Whether this attempt may have taken effect, accounting for unsent request data. - /// Without a frame signal, assume the request may have been sent. - fn attempt_may_have_side_effects(&self, frame_signal: Option<&FrameSignal>) -> bool { - !self.has_no_side_effects() && frame_signal.is_none_or(|s| s.is_signalled()) - } -} - #[derive(Debug, Clone, Copy)] pub struct RetryBackoffBuilder { pub min_base_delay: Duration, diff --git a/sdk/src/session/append.rs b/sdk/src/session/append.rs index 43989a7b..1a133a88 100644 --- a/sdk/src/session/append.rs +++ b/sdk/src/session/append.rs @@ -22,12 +22,10 @@ use crate::{ api::{ApiError, BasinClient, Streaming, retry_builder}, error::{AppendError, RequestError}, frame_signal::FrameSignal, - reconnect::{AdvisedReconnects, ReconnectAdvice}, - retry::{AppendRetryError, RetryBackoffBuilder}, - session::StreamHeaders, + retry::RetryBackoffBuilder, types::{ - AccessTokenMode, AppendAck, AppendInput, AppendRetryPolicy, MeteredBytes, ONE_MIB, - StreamConfig, StreamName, StreamPosition, ValidationError, + AccessTokenMode, AppendAck, AppendInput, AppendRetryPolicy, EncryptionKey, MeteredBytes, + ONE_MIB, StreamName, StreamPosition, ValidationError, }, }; @@ -59,16 +57,6 @@ pub enum AppendSessionError { /// The server returned an invalid append acknowledgement. #[error("invalid append acknowledgement: {0}")] InvalidAck(String), - /// The final attempt failed definitively, but an earlier attempt may have taken effect, - /// so the entire append operation is indeterminate. - #[error( - "append may have taken effect in an earlier attempt; final attempt failed: {final_attempt_error}" - )] - IndefiniteFailure { - /// The definite error returned by the final attempt. - #[source] - final_attempt_error: Box, - }, } impl AppendSessionError { @@ -76,9 +64,6 @@ impl AppendSessionError { pub fn is_retryable(&self) -> bool { match self { Self::Append(error) => error.is_retryable(), - Self::IndefiniteFailure { - final_attempt_error, - } => final_attempt_error.is_retryable(), Self::AckTimeout | Self::ServerDisconnected => true, Self::StreamClosedEarly | Self::SessionClosed @@ -92,7 +77,6 @@ impl AppendSessionError { pub fn has_no_side_effects(&self) -> bool { match self { Self::Append(error) => error.has_no_side_effects(), - Self::IndefiniteFailure { .. } => false, Self::SessionClosed | Self::SessionClosing => true, Self::AckTimeout | Self::ServerDisconnected @@ -106,9 +90,6 @@ impl AppendSessionError { pub fn request_error(&self) -> Option<&RequestError> { match self { Self::Append(error) => error.request_error(), - Self::IndefiniteFailure { - final_attempt_error, - } => final_attempt_error.request_error(), Self::AckTimeout | Self::ServerDisconnected | Self::StreamClosedEarly @@ -125,30 +106,16 @@ impl AppendSessionError { Self::Append(AppendError::Request(error)) if error.is_authentication_error() ) } - - fn is_server_draining(&self) -> bool { - matches!( - self, - Self::Append(AppendError::Request(error)) if error.is_server_draining() - ) - } -} - -impl AppendRetryError for AppendSessionError { - fn has_no_side_effects(&self) -> bool { - Self::has_no_side_effects(self) - } - - fn into_indefinite_failure(self) -> Self { - Self::IndefiniteFailure { - final_attempt_error: Box::new(self), - } - } } impl From for AppendSessionError { fn from(error: ApiError) -> Self { - Self::Append(error.into()) + match error { + ApiError::AppendConditionFailed(condition) => { + Self::Append(AppendError::ConditionFailed(condition.into())) + } + other => Self::Append(AppendError::Request(other.into())), + } } } @@ -179,7 +146,6 @@ impl Future for BatchSubmitTicket { pub struct AppendSessionConfig { max_unacked_bytes: u32, max_unacked_batches: Option, - stream_config: Option, } impl Default for AppendSessionConfig { @@ -187,7 +153,6 @@ impl Default for AppendSessionConfig { Self { max_unacked_bytes: 5 * ONE_MIB, max_unacked_batches: None, - stream_config: None, } } } @@ -222,23 +187,6 @@ impl AppendSessionConfig { ..self } } - - /// Set the stream configuration to apply if the stream is created on append. - /// - /// Unset fields inherit the basin's default stream configuration. Ignored if the stream - /// already exists. - /// - /// Defaults to `None`. - pub fn with_stream_config(self, stream_config: StreamConfig) -> Self { - Self { - stream_config: Some(stream_config), - ..self - } - } - - pub(crate) fn stream_config(&self) -> Option<&StreamConfig> { - self.stream_config.as_ref() - } } struct SessionState { @@ -252,14 +200,6 @@ struct SessionState { stashed_submission: Option, } -impl SessionState { - fn is_close_complete(&self) -> bool { - self.close_tx.is_some() - && self.inflight_appends.is_empty() - && self.stashed_submission.is_none() - } -} - /// A session for high-throughput appending with backpressure control. It can be created from /// [`append_session`](crate::S2Stream::append_session). /// @@ -275,7 +215,7 @@ impl AppendSession { pub(crate) fn new( client: BasinClient, stream: StreamName, - headers: StreamHeaders, + encryption: Option, config: AppendSessionConfig, ) -> Self { let buffer_size = config @@ -289,7 +229,7 @@ impl AppendSession { let handle = AbortOnDropHandle::new(tokio::spawn(run_session_with_retry( client, stream, - headers, + encryption, cmd_rx, retry_builder, buffer_size, @@ -400,7 +340,11 @@ pub(crate) struct AppendSessionInternal { } impl AppendSessionInternal { - pub(crate) fn new(client: BasinClient, stream: StreamName, headers: StreamHeaders) -> Self { + pub(crate) fn new( + client: BasinClient, + stream: StreamName, + encryption: Option, + ) -> Self { let buffer_size = DEFAULT_CHANNEL_BUFFER_SIZE; let (cmd_tx, cmd_rx) = mpsc::channel(buffer_size); let retry_builder = retry_builder(&client.config.retry); @@ -408,7 +352,7 @@ impl AppendSessionInternal { let handle = AbortOnDropHandle::new(tokio::spawn(run_session_with_retry( client, stream, - headers, + encryption, cmd_rx, retry_builder, buffer_size, @@ -513,7 +457,7 @@ impl AppendPermits { async fn run_session_with_retry( client: BasinClient, stream: StreamName, - headers: StreamHeaders, + encryption: Option, cmd_rx: mpsc::Receiver, retry_builder: RetryBackoffBuilder, buffer_size: usize, @@ -537,43 +481,22 @@ async fn run_session_with_retry( }; let mut prev_total_acked_records = 0; let mut retry_backoff = retry_builder.build(); - let mut advised_reconnects = AdvisedReconnects::default(); loop { let result = run_session( &client, &stream, - &headers, + encryption.as_ref(), &mut state, buffer_size, &frame_signal, - advised_reconnects, ) .await; match result { - Ok(SessionOutcome::Closed) => { + Ok(()) => { break; } - Ok(SessionOutcome::ReconnectAdvised) => { - // The advised connection was already poisoned when the advice - // was first decoded, so reconnecting dials a fresh one. - advised_reconnects.record(); - debug!( - inflight_appends_len = state.inflight_appends.len(), - advised_reconnects = advised_reconnects.count(), - "reconnecting append session on server advice" - ); - } - Err(err) if err.is_server_draining() && state.is_close_complete() => break, - Err(err) if err.is_server_draining() => { - advised_reconnects.record(); - debug!( - inflight_appends_len = state.inflight_appends.len(), - advised_reconnects = advised_reconnects.count(), - "reconnecting append session while server drains" - ); - } Err(err) => { if prev_total_acked_records < state.total_acked_records { prev_total_acked_records = state.total_acked_records; @@ -588,11 +511,6 @@ async fn run_session_with_retry( access_token_mode, ) && let Some(backoff) = retry_backoff.next() { - if err.attempt_may_have_side_effects(frame_signal.as_ref()) { - for append in &mut state.inflight_appends { - append.prior_uncertainty = true; - } - } debug!( %err, ?backoff, @@ -607,16 +525,12 @@ async fn run_session_with_retry( "not retrying append session" ); - let session_err = err.clone().with_prior_uncertainty( - state.inflight_appends.iter().any(|a| a.prior_uncertainty), - ); - let _ = terminal_err.set(session_err.clone()); + let err: AppendSessionError = err; + + let _ = terminal_err.set(err.clone()); for inflight_append in state.inflight_appends.drain(..) { - let error = err - .clone() - .with_prior_uncertainty(inflight_append.prior_uncertainty); - let _ = inflight_append.ack_tx.send(Err(error)); + let _ = inflight_append.ack_tx.send(Err(err.clone())); } if let Some(stashed) = state.stashed_submission.take() { @@ -624,16 +538,12 @@ async fn run_session_with_retry( } if let Some(done_tx) = state.close_tx.take() { - let _ = done_tx.send(Err(session_err.clone())); + let _ = done_tx.send(Err(err.clone())); } state.cmd_rx.close(); while let Some(cmd) = state.cmd_rx.recv().await { - let error = match &cmd { - Command::Submit { .. } => &err, - Command::Close { .. } => &session_err, - }; - cmd.reject(error.clone()); + cmd.reject(err.clone()); } break; } @@ -646,35 +556,24 @@ async fn run_session_with_retry( } } -/// How a connection attempt ended without failing. -enum SessionOutcome { - /// Everything submitted was acknowledged and the caller closed the session. - Closed, - /// The server advised reconnecting and this connection drained cleanly. - ReconnectAdvised, -} - async fn run_session( client: &BasinClient, stream: &StreamName, - headers: &StreamHeaders, + encryption: Option<&EncryptionKey>, state: &mut SessionState, buffer_size: usize, frame_signal: &Option, - advised_reconnects: AdvisedReconnects, -) -> Result { +) -> Result<(), AppendSessionError> { if let Some(s) = frame_signal { s.reset(); } - let reconnect = ReconnectAdvice::default(); let (input_tx, mut acks) = connect( client, stream, - headers, + encryption, buffer_size, frame_signal.clone(), - reconnect.clone(), ) .await?; let ack_timeout = client.config.request_timeout; @@ -690,24 +589,10 @@ async fn run_session( assert_eq!(state.inflight_bytes, 0); } - if state.is_close_complete() { - return Ok(SessionOutcome::Closed); - } - let timer = MuxTimer::::default(); tokio::pin!(timer); - let mut declined_advice = false; - loop { - if reconnect.is_advised() && state.close_tx.is_none() && !declined_advice { - if advised_reconnects.should_reconnect() { - drain_for_reconnect(input_tx, acks, state, timer.as_mut(), ack_timeout).await?; - return Ok(SessionOutcome::ReconnectAdvised); - } - declined_advice = true; - } - tokio::select! { (event_ord, _deadline) = &mut timer, if timer.is_armed() => { match TimerEvent::from(event_ord) { @@ -741,7 +626,6 @@ async fn run_session( ack_tx: submission.ack_tx, ack_deadline, _permit: submission.permit, - prior_uncertainty: false, }); } @@ -793,7 +677,10 @@ async fn run_session( } } - if state.is_close_complete() { + if state.close_tx.is_some() + && state.inflight_appends.is_empty() + && state.stashed_submission.is_none() + { break; } } @@ -802,7 +689,7 @@ async fn run_session( assert_eq!(state.inflight_bytes, 0); assert!(state.stashed_submission.is_none()); - Ok(SessionOutcome::Closed) + Ok(()) } async fn resend( @@ -884,68 +771,12 @@ async fn resend( Ok(()) } -/// Half-close so the server acknowledges everything it accepted and then ends -/// the response cleanly. Every input reaches the server ahead of the request's -/// end, so a clean end with appends still unacknowledged is a truncated -/// response, and nothing is resent. -async fn drain_for_reconnect( - input_tx: mpsc::Sender, - mut acks: Streaming, - state: &mut SessionState, - mut timer: Pin<&mut MuxTimer>, - ack_timeout: Duration, -) -> Result<(), AppendSessionError> { - drop(input_tx); - loop { - // Bound the wait for the server's end of stream, which is otherwise - // unbounded once nothing is in flight. - if !timer.is_armed() { - timer.as_mut().fire_at( - TimerEvent::AckDeadline, - Instant::now() + ack_timeout, - CoalesceMode::Earliest, - ); - } - - tokio::select! { - (event_ord, _deadline) = &mut timer, if timer.is_armed() => { - match TimerEvent::from(event_ord) { - TimerEvent::AckDeadline => { - return Err(AppendSessionError::AckTimeout); - } - } - } - - ack = acks.next() => { - match ack { - Some(Ok(ack)) => { - process_ack(ack, state, timer.as_mut())?; - } - Some(Err(err)) if err.is_server_draining() => { - return Ok(()); - } - Some(Err(err)) => { - return Err(err.into()); - } - None => { - if !state.inflight_appends.is_empty() { - return Err(AppendSessionError::StreamClosedEarly); - } - return Ok(()); - } - } - } - } - } -} - async fn connect( client: &BasinClient, stream: &StreamName, - headers: &StreamHeaders, + encryption: Option<&EncryptionKey>, buffer_size: usize, frame_signal: Option, - reconnect: ReconnectAdvice, ) -> Result<(mpsc::Sender, Streaming), AppendSessionError> { let (input_tx, input_rx) = mpsc::channel::(buffer_size); let ack_stream = Box::pin( @@ -953,10 +784,8 @@ async fn connect( .append_session( stream, ReceiverStream::new(input_rx).map(|i| i.into()), - headers.encryption.as_ref(), - headers.stream_config.as_ref(), + encryption, frame_signal, - reconnect, ) .await? .map(|ack| match ack { @@ -1037,7 +866,6 @@ struct InflightAppend { ack_tx: oneshot::Sender>, ack_deadline: Instant, _permit: Option, - prior_uncertainty: bool, } enum Command { @@ -1074,7 +902,9 @@ fn is_safe_to_retry( let policy_compliant = match policy { AppendRetryPolicy::All => true, AppendRetryPolicy::NoSideEffects => { - !has_inflight || !err.attempt_may_have_side_effects(frame_signal) + !has_inflight + || !frame_signal.is_none_or(|s| s.is_signalled()) + || err.has_no_side_effects() } }; policy_compliant @@ -1110,16 +940,13 @@ impl From for TimerEvent { #[cfg(test)] mod tests { - use std::error::Error; - use http::StatusCode; use super::{AppendSessionError, is_safe_to_retry}; use crate::{ api::{ApiError, ServerErrorBody}, - error::{AppendError, ProducerError, RequestError}, + error::{AppendError, RequestError}, frame_signal::FrameSignal, - retry::AppendRetryError, types::{AccessTokenMode, AppendRetryPolicy}, }; @@ -1133,61 +960,6 @@ mod tests { )))) } - #[rstest::rstest] - #[case(StatusCode::FORBIDDEN, "permission_denied", false, false)] - #[case(StatusCode::FORBIDDEN, "permission_denied", true, true)] - #[case(StatusCode::SERVICE_UNAVAILABLE, "unavailable", false, false)] - #[case(StatusCode::SERVICE_UNAVAILABLE, "unavailable", true, false)] - #[case(StatusCode::TOO_MANY_REQUESTS, "rate_limited", true, true)] - #[test] - fn session_failure_preserves_uncertainty_and_latest_error( - #[case] status: StatusCode, - #[case] code: &str, - #[case] prior_uncertainty: bool, - #[case] wrapped: bool, - ) { - let latest = server_error(status, code); - let error = latest.clone().with_prior_uncertainty(prior_uncertainty); - assert_eq!( - matches!(error, AppendSessionError::IndefiniteFailure { .. }), - wrapped - ); - assert_eq!(error.is_retryable(), latest.is_retryable()); - assert_eq!( - error.has_no_side_effects(), - !wrapped && latest.has_no_side_effects() - ); - assert_eq!( - error.request_error().unwrap().server_error().unwrap().code, - code - ); - if wrapped { - let source = error - .source() - .unwrap() - .downcast_ref::>() - .unwrap(); - assert_eq!(source.to_string(), latest.to_string()); - assert!(source.has_no_side_effects()); - } - - let producer_error = ProducerError::from(error.clone()); - assert_eq!(producer_error.is_retryable(), error.is_retryable()); - assert_eq!( - producer_error.has_no_side_effects(), - error.has_no_side_effects() - ); - assert_eq!( - producer_error - .request_error() - .unwrap() - .server_error() - .unwrap() - .code, - code - ); - } - #[test] fn safe_to_retry_session_all_policy() { let retryable = server_error(StatusCode::INTERNAL_SERVER_ERROR, "internal"); diff --git a/sdk/src/session/mod.rs b/sdk/src/session/mod.rs index 7868ed29..107df6e4 100644 --- a/sdk/src/session/mod.rs +++ b/sdk/src/session/mod.rs @@ -5,12 +5,3 @@ pub(crate) use append::{AppendPermit, AppendPermits, AppendSessionInternal, Batc pub use append::{AppendSession, AppendSessionConfig}; pub(crate) use read::read_session; pub use read::{ReadSession, ReadSessionError}; - -/// Per-stream options sent as request headers on every (re)connect of a session. -#[derive(Debug, Clone, Default)] -pub(crate) struct StreamHeaders { - /// `s2-encryption-key` - pub encryption: Option, - /// `s2-stream-config` - pub stream_config: Option, -} diff --git a/sdk/src/session/read.rs b/sdk/src/session/read.rs index 83f848f4..87936cec 100644 --- a/sdk/src/session/read.rs +++ b/sdk/src/session/read.rs @@ -20,11 +20,9 @@ use tracing::debug; use crate::{ api::{ApiError, BasinClient, retry_builder}, error::{ReadError, RequestError}, - reconnect::{AdvisedReconnects, ReconnectAdvice}, retry::RetryBackoff, - session::StreamHeaders, types::{ - AccessTokenMode, MeteredBytes, ReadBatch, ReadInput, ReadSessionConfig, + AccessTokenMode, EncryptionKey, MeteredBytes, ReadBatch, ReadInput, ReadSessionConfig, ReadSessionRetryPolicy, StreamName, StreamPosition, }, }; @@ -48,10 +46,6 @@ impl ReadSessionFailure { fn is_authentication_error(&self) -> bool { matches!(self, Self::Api(error) if error.is_authentication_error()) } - - fn is_server_draining(&self) -> bool { - matches!(self, Self::Api(error) if error.is_server_draining()) - } } /// Errors returned by a read session. @@ -93,23 +87,9 @@ impl From for ReadSessionError { } } -/// The server heartbeats a tailing read session at a randomized gap of at most -/// 15 seconds (), plus some buffer. -const HEARTBEAT_TIMEOUT: Duration = Duration::from_secs(20); - type InternalStreaming = Pin>>>; -/// An item from a single read connection. -enum ReadItem { - Batch(ReadBatch), - /// The server advised reconnecting and the response ended cleanly. - /// - /// Always the last item of a connection, emitted after the batch it rode - /// in on, so the resume position already accounts for that batch. - ReconnectAdvised, -} - #[derive(Debug, Clone, thiserror::Error)] #[non_exhaustive] /// Error returned while waiting for a read session to catch up. @@ -372,7 +352,7 @@ impl Drop for ReadSession { pub async fn read_session( client: BasinClient, name: StreamName, - headers: StreamHeaders, + encryption: Option, input: ReadInput, config: ReadSessionConfig, ) -> Result { @@ -380,7 +360,6 @@ pub async fn read_session( start, stop, ignore_command_records, - stream_config: _, } = input; let mut start: ReadStart = start.into(); let mut end: ReadEnd = stop.into(); @@ -389,7 +368,6 @@ pub async fn read_session( let access_token_mode = client.config.access_token.mode(); let baseline_wait = end.wait; let mut last_tail_at: Option = None; - let mut advised_reconnects = AdvisedReconnects::default(); let initial_resume_seq_num = if start.clamp == Some(true) { None } else { @@ -401,11 +379,9 @@ pub async fn read_session( match session_inner( client.clone(), name.clone(), - headers.clone(), + encryption.clone(), start.clone(), end.clone(), - ReconnectAdvice::default(), - advised_reconnects, ) .await { @@ -414,13 +390,6 @@ pub async fn read_session( break batches; } Err(err) => { - if take_server_draining_reconnect(&err, &mut advised_reconnects) { - debug!( - advised_reconnects = advised_reconnects.count(), - "reconnecting initial read session while server drains" - ); - continue; - } if let Some(backoff) = retry_delay(&err, &mut retry_backoff, retry_policy, access_token_mode) { @@ -433,7 +402,7 @@ pub async fn read_session( }; let updates = Box::pin(stream! { - let mut batches: Option> = Some(batches); + let mut batches: Option> = Some(batches); loop { if batches.is_none() { @@ -441,22 +410,12 @@ pub async fn read_session( match session_inner( client.clone(), name.clone(), - headers.clone(), + encryption.clone(), start.clone(), end.clone(), - ReconnectAdvice::default(), - advised_reconnects, ).await { Ok(b) => batches = Some(b), Err(err) => { - if take_server_draining_reconnect(&err, &mut advised_reconnects) { - debug!( - resume_seq_num = ?start.seq_num, - advised_reconnects = advised_reconnects.count(), - "reconnecting read session while server drains" - ); - continue; - } if let Some(backoff) = retry_delay( &err, @@ -480,25 +439,7 @@ pub async fn read_session( .next() .await { - Some(Ok(ReadItem::ReconnectAdvised)) => { - batches = None; - // The advised connection was already poisoned when the - // advice was first decoded; reconnecting dials a fresh - // one. Avoid a useless reconnect for a read that was - // already satisfied when the advice arrived. - if read_limits_exhausted(&end) { - break; - } - advised_reconnects.record(); - debug!( - resume_seq_num = ?start.seq_num, - advised_reconnects = advised_reconnects.count(), - "reconnecting read session on server advice" - ); - yield Ok(ReadUpdate::behind()); - continue; - } - Some(Ok(ReadItem::Batch(batch))) => { + Some(Ok(batch)) => { if retry_backoff.used() > 0 { retry_backoff.reset(); } @@ -521,18 +462,6 @@ pub async fn read_session( } Some(Err(err)) => { batches = None; - if err.is_server_draining() && read_limits_exhausted(&end) { - break; - } - if take_server_draining_reconnect(&err, &mut advised_reconnects) { - debug!( - resume_seq_num = ?start.seq_num, - advised_reconnects = advised_reconnects.count(), - "reconnecting read session while server drains" - ); - yield Ok(ReadUpdate::behind()); - continue; - } if let Some(backoff) = retry_delay( &err, @@ -581,36 +510,18 @@ fn update_resume_start(start: &mut ReadStart, batch: &ReadBatch) { async fn session_inner( client: BasinClient, name: StreamName, - headers: StreamHeaders, + encryption: Option, start: ReadStart, end: ReadEnd, - reconnect: ReconnectAdvice, - advised_reconnects: AdvisedReconnects, -) -> Result, ReadSessionFailure> { +) -> Result, ReadSessionFailure> { let mut batches = client - .read_session( - &name, - start, - end, - headers.encryption.as_ref(), - headers.stream_config.as_ref(), - reconnect.clone(), - ) + .read_session(&name, start, end, encryption.as_ref()) .await?; - - let mut declined_advice = false; Ok(Box::pin(try_stream! { loop { - match timeout(HEARTBEAT_TIMEOUT, batches.next()).await { + match timeout(Duration::from_secs(20), batches.next()).await { Ok(Some(batch)) => { - yield ReadItem::Batch(ReadBatch::from_api(batch?)); - if reconnect.is_advised() && !declined_advice { - if advised_reconnects.should_reconnect() { - yield ReadItem::ReconnectAdvised; - break; - } - declined_advice = true; - } + yield ReadBatch::from_api(batch?); } Ok(None) => break, Err(_) => Err(ReadSessionFailure::HeartbeatTimeout)?, @@ -619,11 +530,6 @@ async fn session_inner( })) } -/// Whether the read's `count` or `bytes` limit has been used up. -fn read_limits_exhausted(end: &ReadEnd) -> bool { - end.count == Some(0) || end.bytes == Some(0) -} - /// Compute the remaining wait budget for a retry. /// /// During catchup (tail not yet observed), the full wait is sent. @@ -679,18 +585,6 @@ fn retry_delay( } } -fn take_server_draining_reconnect( - err: &ReadSessionFailure, - advised_reconnects: &mut AdvisedReconnects, -) -> bool { - if err.is_server_draining() { - advised_reconnects.record(); - true - } else { - false - } -} - #[cfg(test)] mod tests { use bytes::Bytes; diff --git a/sdk/src/types.rs b/sdk/src/types.rs index 099048d2..97b71416 100644 --- a/sdk/src/types.rs +++ b/sdk/src/types.rs @@ -15,9 +15,7 @@ use std::{ #[cfg(feature = "_hidden")] use async_trait::async_trait; use bytes::Bytes; -use compact_str::CompactString; use http::{ - HeaderMap, header::HeaderValue, uri::{Authority, Scheme}, }; @@ -27,12 +25,11 @@ use s2_api::{v1 as api, v1::stream::s2s::CompressionAlgorithm}; pub use s2_common::ValidationError; /// Access token ID. /// -/// **Note:** It must be unique to the account and between 1 and 96 bytes in length, and must -/// not contain NUL bytes. +/// **Note:** It must be unique to the account and between 1 and 96 bytes in length. pub use s2_common::access::AccessTokenId; -/// See [`ListAccessTokensInput::prefix`]. It must not contain NUL bytes. +/// See [`ListAccessTokensInput::prefix`]. pub use s2_common::access::AccessTokenIdPrefix; -/// See [`ListAccessTokensInput::start_after`]. It must not contain NUL bytes. +/// See [`ListAccessTokensInput::start_after`]. pub use s2_common::access::AccessTokenIdStartAfter; /// Basin name. /// @@ -50,12 +47,11 @@ pub use s2_common::basin::BasinNameStartAfter; pub use s2_common::location::LocationName; /// Stream name. /// -/// **Note:** It must be unique to the basin and between 1 and 512 bytes in length, and must -/// not contain NUL bytes. +/// **Note:** It must be unique to the basin and between 1 and 512 bytes in length. pub use s2_common::stream::StreamName; -/// See [`ListStreamsInput::prefix`]. It must not contain NUL bytes. +/// See [`ListStreamsInput::prefix`]. pub use s2_common::stream::StreamNamePrefix; -/// See [`ListStreamsInput::start_after`]. It must not contain NUL bytes. +/// See [`ListStreamsInput::start_after`]. pub use s2_common::stream::StreamNameStartAfter; pub use s2_common::{ caps::RECORD_BATCH_MAX, @@ -513,7 +509,6 @@ pub struct S2Config { pub(crate) retry: RetryConfig, pub(crate) compression: Compression, pub(crate) user_agent: HeaderValue, - pub(crate) default_headers: HeaderMap, pub(crate) insecure_skip_cert_verification: bool, pub(crate) rustls_crypto_provider: Option>, } @@ -531,7 +526,6 @@ impl S2Config { user_agent: concat!("s2-sdk-rust/", env!("CARGO_PKG_VERSION")) .parse() .expect("valid user agent"), - default_headers: HeaderMap::new(), insecure_skip_cert_verification: false, rustls_crypto_provider: default_rustls_crypto_provider(), } @@ -551,54 +545,6 @@ impl S2Config { Self { endpoints, ..self } } - /// Set additional HTTP headers to send with every request. - /// - /// These headers apply to account, basin, and stream operations, including - /// retries and streaming requests. SDK-generated headers, such as - /// authorization and basin routing, take precedence over these defaults. - /// Calling this method again replaces the previous set of default headers. - /// - /// `Accept-Encoding` defaults are used only when [`Compression::None`] is - /// configured; otherwise the SDK sets the header to the configured - /// compression algorithm. - /// - /// Headers are sent to all configured S2 endpoints. Use - /// [`HeaderValue::set_sensitive`] for values that should be redacted in debug - /// output. Do not use these defaults for per-request identifiers, since the - /// same values are reused across requests. - /// - /// # Errors - /// - /// Returns an error if `default_headers` contains `Content-Type`, - /// `Content-Encoding`, `Content-Length`, or `Transfer-Encoding`. - /// The SDK controls request format and body framing. - /// Use [`Self::with_compression`] to configure request body encoding. - #[cfg(feature = "_hidden")] - #[doc(hidden)] - pub fn with_default_headers(self, default_headers: HeaderMap) -> Result { - if default_headers.contains_key(http::header::CONTENT_ENCODING) { - return Err(ValidationError( - "Content-Encoding cannot be set in default headers; use S2Config::with_compression instead" - .into(), - )); - } - for name in [ - http::header::CONTENT_TYPE, - http::header::CONTENT_LENGTH, - http::header::TRANSFER_ENCODING, - ] { - if default_headers.contains_key(&name) { - return Err(ValidationError(format!( - "{name} cannot be set in default headers; the SDK controls request format and body framing" - ))); - } - } - Ok(Self { - default_headers, - ..self - }) - } - /// Set the timeout for establishing a connection to the server. /// /// Defaults to `3s`. @@ -734,6 +680,33 @@ impl Page { } } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// Storage class for recent appends. +pub enum StorageClass { + /// Standard storage class that offers append latencies under `500ms`. + Standard, + /// Express storage class that offers append latencies under `50ms`. + Express, +} + +impl From for StorageClass { + fn from(value: api::config::StorageClass) -> Self { + match value { + api::config::StorageClass::Standard => StorageClass::Standard, + api::config::StorageClass::Express => StorageClass::Express, + } + } +} + +impl From for api::config::StorageClass { + fn from(value: StorageClass) -> Self { + match value { + StorageClass::Standard => api::config::StorageClass::Standard, + StorageClass::Express => api::config::StorageClass::Express, + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] /// Retention policy for records in a stream. pub enum RetentionPolicy { @@ -893,8 +866,10 @@ impl From for api::config::DeleteOnEmptyConfig { #[non_exhaustive] /// Configuration for a stream. pub struct StreamConfig { - /// [Storage class](https://s2.dev/docs/storage-classes) for the stream. - pub storage_class: Option, + /// Storage class for the stream. + /// + /// Defaults to [`Express`](StorageClass::Express). + pub storage_class: Option, /// Retention policy for records in the stream. /// /// Defaults to `7 days` of retention. @@ -916,9 +891,9 @@ impl StreamConfig { } /// Set the storage class for the stream. - pub fn with_storage_class(self, storage_class: impl Into) -> Self { + pub fn with_storage_class(self, storage_class: StorageClass) -> Self { Self { - storage_class: Some(storage_class.into()), + storage_class: Some(storage_class), ..self } } @@ -951,7 +926,7 @@ impl StreamConfig { impl From for StreamConfig { fn from(value: api::config::StreamConfig) -> Self { Self { - storage_class: value.storage_class, + storage_class: value.storage_class.map(Into::into), retention_policy: value.retention_policy.map(Into::into), timestamping: value.timestamping.map(Into::into), delete_on_empty: value.delete_on_empty.map(Into::into), @@ -962,7 +937,7 @@ impl From for StreamConfig { impl From for api::config::StreamConfig { fn from(value: StreamConfig) -> Self { Self { - storage_class: value.storage_class, + storage_class: value.storage_class.map(Into::into), retention_policy: value.retention_policy.map(Into::into), timestamping: value.timestamping.map(Into::into), delete_on_empty: value.delete_on_empty.map(Into::into), @@ -1440,7 +1415,7 @@ impl From for api::config::DeleteOnEmptyReconfigur /// Reconfiguration for [`StreamConfig`]. pub struct StreamReconfiguration { /// Override for the existing [`storage_class`](StreamConfig::storage_class). - pub storage_class: Maybe>, + pub storage_class: Maybe>, /// Override for the existing [`retention_policy`](StreamConfig::retention_policy). pub retention_policy: Maybe>, /// Override for the existing [`timestamping`](StreamConfig::timestamping). @@ -1456,9 +1431,9 @@ impl StreamReconfiguration { } /// Set the override for the existing [`storage_class`](StreamConfig::storage_class). - pub fn with_storage_class(self, storage_class: impl Into) -> Self { + pub fn with_storage_class(self, storage_class: StorageClass) -> Self { Self { - storage_class: Maybe::Specified(Some(storage_class.into())), + storage_class: Maybe::Specified(Some(storage_class)), ..self } } @@ -1491,7 +1466,7 @@ impl StreamReconfiguration { impl From for api::config::StreamReconfiguration { fn from(value: StreamReconfiguration) -> Self { Self { - storage_class: value.storage_class, + storage_class: value.storage_class.map(|m| m.map(Into::into)), retention_policy: value.retention_policy.map(|m| m.map(Into::into)), timestamping: value.timestamping.map(|m| m.map(Into::into)), delete_on_empty: value.delete_on_empty.map(|m| m.map(Into::into)), @@ -1683,10 +1658,6 @@ pub struct LocationInfo { pub name: LocationName, /// Location represents a private placement, limited by account. pub is_private: bool, - /// [Storage classes](https://s2.dev/docs/storage-classes) available to the account in this location. - pub storage_classes: Option>, - /// Default [storage class](https://s2.dev/docs/storage-classes) for this location. - pub default_storage_class: Option, } impl From for LocationInfo { @@ -1694,8 +1665,6 @@ impl From for LocationInfo { Self { name: value.name, is_private: value.is_private, - storage_classes: value.storage_classes, - default_storage_class: value.default_storage_class, } } } @@ -2433,7 +2402,7 @@ pub enum BasinMetricSet { /// Returns a [`GaugeMetric`] representing a timeseries of total stored bytes across all streams /// in the basin, with one observed value for each hour over the requested time range. Storage(TimeRange), - /// Returns [`AccumulationMetric`]s, one per storage class. + /// Returns [`AccumulationMetric`]s, one per storage class (standard, express). /// /// Each metric represents a timeseries of the number of append operations across all streams /// in the basin, with one accumulated value per interval over the requested time range. @@ -3391,16 +3360,6 @@ pub struct AppendInput { /// If unspecified, no matching is performed. If specified and mismatched, /// the append fails. A stream defaults to `""` as its fencing token. pub fencing_token: Option, - /// Stream configuration to apply if the stream is created on append. - /// - /// Unset fields inherit the basin's default stream configuration. Ignored if the stream - /// already exists. - /// - /// Only used by [`append`](crate::S2Stream::append). Append sessions send the header once - /// at connect; see - /// [`AppendSessionConfig::with_stream_config`](crate::append_session::AppendSessionConfig::with_stream_config) - /// and [`ProducerConfig::with_stream_config`](crate::producer::ProducerConfig::with_stream_config). - pub stream_config: Option, } impl AppendInput { @@ -3410,15 +3369,6 @@ impl AppendInput { records, match_seq_num: None, fencing_token: None, - stream_config: None, - } - } - - /// Set the stream configuration to apply if the stream is created on append. - pub fn with_stream_config(self, stream_config: StreamConfig) -> Self { - Self { - stream_config: Some(stream_config), - ..self } } @@ -3675,11 +3625,6 @@ pub struct ReadInput { /// /// Defaults to `false`. pub ignore_command_records: bool, - /// Stream configuration to apply if the stream is created on read. - /// - /// Unset fields inherit the basin's default stream configuration. Ignored if the stream - /// already exists. - pub stream_config: Option, } #[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] @@ -3747,14 +3692,6 @@ impl ReadInput { ..self } } - - /// Set the stream configuration to apply if the stream is created on read. - pub fn with_stream_config(self, stream_config: StreamConfig) -> Self { - Self { - stream_config: Some(stream_config), - ..self - } - } } #[derive(Debug, Clone)] @@ -4050,51 +3987,15 @@ mod tests { assert!(!cfg.insecure_skip_cert_verification); } - #[cfg(feature = "_hidden")] - #[rstest] - #[case::matching_compression("content-encoding", "gzip", Compression::Gzip)] - #[case::mixed_case("Content-Encoding", "identity", Compression::None)] - #[case::empty_value("content-encoding", "", Compression::None)] - fn default_headers_reject_content_encoding( - #[case] name: &str, - #[case] value: &str, - #[case] compression: Compression, - ) { - let headers = HeaderMap::from_iter([( - name.parse::().unwrap(), - HeaderValue::from_str(value).unwrap(), - )]); - let error = S2Config::new("token") - .with_compression(compression) - .with_default_headers(headers) - .unwrap_err(); - assert!(error.0.contains("Content-Encoding")); - assert!(error.0.contains("with_compression")); - } + // -- StorageClass -- - #[cfg(feature = "_hidden")] #[rstest] - #[case::content_type_s2s("content-type", "s2s/proto")] - #[case::content_type_protobuf("content-type", "application/protobuf")] - #[case::content_type_json("content-type", "application/json")] - #[case::content_type_mixed_case("Content-Type", "s2s/proto")] - #[case::content_type_empty("content-type", "")] - #[case::content_length("content-length", "123")] - #[case::content_length_mixed_case("Content-Length", "0")] - #[case::content_length_empty("content-length", "")] - #[case::transfer_encoding("transfer-encoding", "chunked")] - #[case::transfer_encoding_mixed_case("Transfer-Encoding", "chunked")] - #[case::transfer_encoding_empty("transfer-encoding", "")] - fn default_headers_reject_framing_headers(#[case] name: &str, #[case] value: &str) { - let headers = HeaderMap::from_iter([( - name.parse::().unwrap(), - HeaderValue::from_str(value).unwrap(), - )]); - let error = S2Config::new("token") - .with_default_headers(headers) - .unwrap_err(); - assert!(error.0.contains(&name.to_ascii_lowercase())); - assert!(error.0.contains("framing")); + #[case::standard(StorageClass::Standard)] + #[case::express(StorageClass::Express)] + fn storage_class_roundtrip(#[case] sdk: StorageClass) { + let api: api::config::StorageClass = sdk.into(); + let back: StorageClass = api.into(); + assert_eq!(back, sdk); } // -- RetentionPolicy -- @@ -4158,7 +4059,7 @@ mod tests { #[test] fn stream_config_builder_and_roundtrip() { let sdk = StreamConfig::new() - .with_storage_class("express") + .with_storage_class(StorageClass::Express) .with_retention_policy(RetentionPolicy::Age(86400)) .with_timestamping(TimestampingConfig { mode: Some(TimestampingMode::ClientPrefer), @@ -4175,7 +4076,9 @@ mod tests { #[test] fn basin_config_builder_and_roundtrip() { let sdk = BasinConfig::new() - .with_default_stream_config(StreamConfig::new().with_storage_class("standard")) + .with_default_stream_config( + StreamConfig::new().with_storage_class(StorageClass::Standard), + ) .with_create_stream_on_append(true) .with_create_stream_on_read(false); let api: api::config::BasinConfig = sdk.clone().into(); diff --git a/sdk/tests/account_ops.rs b/sdk/tests/account_ops.rs index 4576a36a..552445d7 100644 --- a/sdk/tests/account_ops.rs +++ b/sdk/tests/account_ops.rs @@ -70,7 +70,7 @@ async fn basin_config_roundtrip() -> Result<(), Box> { let config = BasinConfig::new() .with_default_stream_config( StreamConfig::new() - .with_storage_class("express") + .with_storage_class(StorageClass::Express) .with_delete_on_empty( DeleteOnEmptyConfig::new().with_min_age(Duration::from_secs(60)), ), @@ -85,7 +85,7 @@ async fn basin_config_roundtrip() -> Result<(), Box> { retrieved_config, BasinConfig { default_stream_config: Some(StreamConfig { - storage_class: Some(ref storage_class), + storage_class: Some(StorageClass::Express), delete_on_empty: Some(DeleteOnEmptyConfig { min_age_secs: 60, .. @@ -94,7 +94,7 @@ async fn basin_config_roundtrip() -> Result<(), Box> { }), create_stream_on_read: true, .. - } if storage_class == "express" + } ); s2.delete_basin(DeleteBasinInput::new(basin_name)).await?; @@ -114,7 +114,9 @@ async fn reconfigure_basin() -> Result<(), Box> { .await?; let new_config = BasinReconfiguration::new() - .with_default_stream_config(StreamReconfiguration::new().with_storage_class("standard")) + .with_default_stream_config( + StreamReconfiguration::new().with_storage_class(StorageClass::Standard), + ) .with_create_stream_on_append(false); let updated_config = s2 @@ -125,12 +127,12 @@ async fn reconfigure_basin() -> Result<(), Box> { updated_config, BasinConfig { default_stream_config: Some(StreamConfig { - storage_class: Some(ref storage_class), + storage_class: Some(StorageClass::Standard), .. }), create_stream_on_append: false, .. - } if storage_class == "standard" + } ); s2.delete_basin(DeleteBasinInput::new(basin_name)).await?; @@ -337,7 +339,7 @@ async fn get_basin_config() -> Result<(), Box> { let basin_name = unique_basin_name(); let config = BasinConfig::new() - .with_default_stream_config(StreamConfig::new().with_storage_class("express")); + .with_default_stream_config(StreamConfig::new().with_storage_class(StorageClass::Express)); s2.create_basin(CreateBasinInput::new(basin_name.clone()).with_config(config)) .await?; @@ -347,9 +349,9 @@ async fn get_basin_config() -> Result<(), Box> { assert_matches!( retrieved_config.default_stream_config, Some(StreamConfig { - storage_class: Some(ref storage_class), + storage_class: Some(StorageClass::Express), .. - }) if storage_class == "express" + }) ); s2.delete_basin(DeleteBasinInput::new(basin_name)).await?; diff --git a/sdk/tests/basin_ops.rs b/sdk/tests/basin_ops.rs index fc9b2cc4..39506e6b 100644 --- a/sdk/tests/basin_ops.rs +++ b/sdk/tests/basin_ops.rs @@ -55,7 +55,7 @@ async fn create_list_and_delete_stream(basin: &S2Basin) -> Result<(), RequestErr async fn stream_config_roundtrip(basin: &S2Basin) -> Result<(), RequestError> { let stream_name = unique_stream_name(); let config = StreamConfig::new() - .with_storage_class("standard") + .with_storage_class(StorageClass::Standard) .with_retention_policy(RetentionPolicy::Age(3600)) .with_timestamping(TimestampingConfig::new().with_mode(TimestampingMode::ClientRequire)); @@ -68,14 +68,14 @@ async fn stream_config_roundtrip(basin: &S2Basin) -> Result<(), RequestError> { assert_matches!( retrieved_config, StreamConfig { - storage_class: Some(ref storage_class), + storage_class: Some(StorageClass::Standard), retention_policy: Some(RetentionPolicy::Age(3600)), timestamping: Some(TimestampingConfig { mode: Some(TimestampingMode::ClientRequire), .. }), .. - } if storage_class == "standard" + } ); Ok(()) @@ -348,7 +348,7 @@ async fn delete_nonexistent_stream_with_ignore(basin: &S2Basin) -> Result<(), Re async fn get_stream_config(basin: &S2Basin) -> Result<(), RequestError> { let stream_name = unique_stream_name(); - let config = StreamConfig::new().with_storage_class("express"); + let config = StreamConfig::new().with_storage_class(StorageClass::Express); basin .create_stream(CreateStreamInput::new(stream_name.clone()).with_config(config)) @@ -356,7 +356,7 @@ async fn get_stream_config(basin: &S2Basin) -> Result<(), RequestError> { let retrieved_config = basin.get_stream_config(stream_name.clone()).await?; - assert_matches!(retrieved_config.storage_class.as_deref(), Some("express")); + assert_matches!(retrieved_config.storage_class, Some(StorageClass::Express)); Ok(()) } @@ -605,7 +605,7 @@ async fn list_all_streams_include_deleted( async fn create_stream_with_full_config(basin: &SharedS2Basin) -> Result<(), RequestError> { let stream_name = unique_stream_name(); let config = StreamConfig::new() - .with_storage_class("standard") + .with_storage_class(StorageClass::Standard) .with_retention_policy(RetentionPolicy::Age(86400)) .with_timestamping( TimestampingConfig::new() @@ -623,7 +623,7 @@ async fn create_stream_with_full_config(basin: &SharedS2Basin) -> Result<(), Req assert_matches!( retrieved, StreamConfig { - storage_class: Some(ref storage_class), + storage_class: Some(StorageClass::Standard), retention_policy: Some(RetentionPolicy::Age(86400)), timestamping: Some(TimestampingConfig { mode: Some(TimestampingMode::ClientRequire), @@ -635,7 +635,7 @@ async fn create_stream_with_full_config(basin: &SharedS2Basin) -> Result<(), Req .. }), .. - } if storage_class == "standard" + } ); basin @@ -649,7 +649,7 @@ async fn create_stream_with_full_config(basin: &SharedS2Basin) -> Result<(), Req #[tokio_shared_rt::test(shared)] async fn create_stream_storage_class_express(basin: &SharedS2Basin) -> Result<(), RequestError> { let stream_name = unique_stream_name(); - let config = StreamConfig::new().with_storage_class("express"); + let config = StreamConfig::new().with_storage_class(StorageClass::Express); let result = basin .create_stream(CreateStreamInput::new(stream_name.clone()).with_config(config)) @@ -664,7 +664,7 @@ async fn create_stream_storage_class_express(basin: &SharedS2Basin) -> Result<() assert_eq!(info.name, stream_name); let retrieved = basin.get_stream_config(stream_name.clone()).await?; - assert_matches!(retrieved.storage_class.as_deref(), Some("express") | None); + assert_matches!(retrieved.storage_class, Some(StorageClass::Express) | None); basin .delete_stream(DeleteStreamInput::new(stream_name)) @@ -853,11 +853,11 @@ async fn reconfigure_stream_storage_class_standard( let config = basin .reconfigure_stream(ReconfigureStreamInput::new( stream_name.clone(), - StreamReconfiguration::new().with_storage_class("standard"), + StreamReconfiguration::new().with_storage_class(StorageClass::Standard), )) .await?; - assert_matches!(config.storage_class.as_deref(), Some("standard")); + assert_matches!(config.storage_class, Some(StorageClass::Standard)); basin .delete_stream(DeleteStreamInput::new(stream_name)) @@ -879,7 +879,7 @@ async fn reconfigure_stream_storage_class_express( let result = basin .reconfigure_stream(ReconfigureStreamInput::new( stream_name.clone(), - StreamReconfiguration::new().with_storage_class("express"), + StreamReconfiguration::new().with_storage_class(StorageClass::Express), )) .await; @@ -894,7 +894,7 @@ async fn reconfigure_stream_storage_class_express( Err(err) => return Err(err), }; - assert_matches!(config.storage_class.as_deref(), Some("express") | None); + assert_matches!(config.storage_class, Some(StorageClass::Express) | None); basin .delete_stream(DeleteStreamInput::new(stream_name)) @@ -1126,7 +1126,7 @@ async fn reconfigure_stream_empty_config_no_change( .create_stream( CreateStreamInput::new(stream_name.clone()).with_config( StreamConfig::new() - .with_storage_class("standard") + .with_storage_class(StorageClass::Standard) .with_retention_policy(RetentionPolicy::Age(3600)), ), ) @@ -1140,7 +1140,7 @@ async fn reconfigure_stream_empty_config_no_change( .await?; let retrieved = basin.get_stream_config(stream_name.clone()).await?; - assert_matches!(retrieved.storage_class.as_deref(), Some("standard")); + assert_matches!(retrieved.storage_class, Some(StorageClass::Standard)); assert_matches!(retrieved.retention_policy, Some(RetentionPolicy::Age(3600))); basin @@ -1229,7 +1229,7 @@ async fn reconfigure_stream_nonexistent_errors(basin: &SharedS2Basin) -> Result< let result = basin .reconfigure_stream(ReconfigureStreamInput::new( unique_stream_name(), - StreamReconfiguration::new().with_storage_class("standard"), + StreamReconfiguration::new().with_storage_class(StorageClass::Standard), )) .await; diff --git a/sdk/tests/stream_ops.rs b/sdk/tests/stream_ops.rs index feb7cad7..54d81180 100644 --- a/sdk/tests/stream_ops.rs +++ b/sdk/tests/stream_ops.rs @@ -3,7 +3,7 @@ mod common; use std::time::Duration; use assert_matches::assert_matches; -use common::{S2Stream, SharedS2Basin, s2, s2_config, unique_basin_name, unique_stream_name}; +use common::{S2Stream, SharedS2Basin, s2_config, unique_basin_name, unique_stream_name}; use futures_util::{StreamExt, poll}; use rstest::rstest; use s2_sdk::{ @@ -1971,7 +1971,7 @@ async fn create_stream_inherits_basin_default_config() -> Result<(), Box Result<(), Box Result<(), Box> { - let s2 = s2(); - let basin_name = unique_basin_name(); - s2.create_basin( - CreateBasinInput::new(basin_name.clone()).with_config( - BasinConfig::new() - .with_create_stream_on_append(true) - .with_default_stream_config(StreamConfig::new().with_storage_class("standard")), - ), - ) - .await?; - let basin = s2.basin(basin_name.clone()); - - let stream_config = StreamConfig::new() - .with_retention_policy(RetentionPolicy::Age(3600)) - .with_delete_on_empty(DeleteOnEmptyConfig::new().with_min_age(Duration::from_secs(300))); - - let unary_stream = unique_stream_name(); - basin - .stream(unary_stream.clone()) - .append( - AppendInput::new(AppendRecordBatch::try_from_iter([AppendRecord::new( - "hello", - )?])?) - .with_stream_config(stream_config.clone()), - ) - .await?; - let config = basin.get_stream_config(unary_stream).await?; - assert_matches!( - config, - StreamConfig { - storage_class: Some(ref storage_class), - retention_policy: Some(RetentionPolicy::Age(3600)), - delete_on_empty: Some(DeleteOnEmptyConfig { - min_age_secs: 300, - .. - }), - .. - } if storage_class == "standard" - ); - - let session_stream = unique_stream_name(); - let producer = basin - .stream(session_stream.clone()) - .producer(ProducerConfig::new().with_stream_config(stream_config.clone())); - producer.submit(AppendRecord::new("hello")?).await?.await?; - producer.close().await?; - let config = basin.get_stream_config(session_stream).await?; - assert_matches!( - config, - StreamConfig { - retention_policy: Some(RetentionPolicy::Age(3600)), - delete_on_empty: Some(DeleteOnEmptyConfig { - min_age_secs: 300, - .. - }), - .. - } - ); - - let existing_stream = unique_stream_name(); - basin - .create_stream(CreateStreamInput::new(existing_stream.clone())) - .await?; - let before = basin.get_stream_config(existing_stream.clone()).await?; - basin - .stream(existing_stream.clone()) - .append( - AppendInput::new(AppendRecordBatch::try_from_iter([AppendRecord::new( - "hello", - )?])?) - .with_stream_config(stream_config), - ) - .await?; - let after = basin.get_stream_config(existing_stream).await?; - assert_eq!(after, before); - assert_ne!(after.retention_policy, Some(RetentionPolicy::Age(3600))); - - s2.delete_basin(DeleteBasinInput::new(basin_name)).await?; - Ok(()) -} - -#[tokio::test] -async fn stream_config_applies_when_read_creates_stream() -> Result<(), Box> -{ - let s2 = s2(); - let basin_name = unique_basin_name(); - s2.create_basin( - CreateBasinInput::new(basin_name.clone()) - .with_config(BasinConfig::new().with_create_stream_on_read(true)), - ) - .await?; - let basin = s2.basin(basin_name.clone()); - - let stream_config = StreamConfig::new().with_retention_policy(RetentionPolicy::Age(3600)); - - let session_stream = unique_stream_name(); - let _session = basin - .stream(session_stream.clone()) - .read_session( - ReadInput::new().with_stream_config(stream_config), - ReadSessionConfig::default(), - ) - .await?; - let config = basin.get_stream_config(session_stream).await?; - assert_eq!(config.retention_policy, Some(RetentionPolicy::Age(3600))); - - s2.delete_basin(DeleteBasinInput::new(basin_name)).await?; - Ok(()) -} diff --git a/sim/Cargo.lock b/sim/Cargo.lock index 19ae3e7f..49a02bc0 100644 --- a/sim/Cargo.lock +++ b/sim/Cargo.lock @@ -20,9 +20,9 @@ dependencies = [ [[package]] name = "aegis" -version = "0.9.15" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58541132f980da31e9aa99f7bdee69bc84bf1e168b9b91ef2dbe8abb7b4ce5dd" +checksum = "e07d39d15384924b35b70d7b8fa1f9a2934101dd3fa4722ede163cc4f9b7b960" dependencies = [ "cc", "softaes", @@ -30,36 +30,48 @@ dependencies = [ [[package]] name = "aes" -version = "0.9.3" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" +checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138" dependencies = [ "cipher", "cpubits", - "cpufeatures 0.3.1", - "zeroize", + "cpufeatures 0.3.0", ] [[package]] name = "aes-gcm" -version = "0.11.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f" +checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" dependencies = [ "aead", "aes", "cipher", "ctr", - "ctutils", "ghash", + "subtle", "zeroize", ] [[package]] -name = "aho-corasick" -version = "1.1.5" +name = "ahash" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -78,9 +90,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" dependencies = [ "libc", ] @@ -121,7 +133,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -132,7 +144,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -145,7 +157,7 @@ dependencies = [ "libloading", "linkme", "once_cell", - "rand 0.8.8", + "rand 0.8.6", "rand_core 0.6.4", "rustc_version_runtime", "serde", @@ -154,24 +166,30 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.104" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "arc-swap" -version = "1.9.2" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" dependencies = [ "rustversion", ] [[package]] -name = "arrayvec" -version = "0.7.8" +name = "arrayref" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "asn1-rs" @@ -185,7 +203,7 @@ dependencies = [ "nom 7.1.3", "num-traits", "rusticata-macros", - "thiserror 2.0.20", + "thiserror 2.0.18", "time", ] @@ -197,7 +215,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", "synstructure", ] @@ -209,7 +227,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -226,9 +244,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.46" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f10dafd0c8d2e51ae9a748805777613ed0bbe17bf586b76c8311f45c020a32f" +checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" dependencies = [ "compression-codecs", "compression-core", @@ -255,26 +273,20 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "async-trait" -version = "0.1.92" +version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] -[[package]] -name = "asyncband" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2d85fd3d291fabcc40c7232c92c280ec1754fd7b5d7ea769f143222143e179a" - [[package]] name = "atoi" version = "3.1.0" @@ -307,9 +319,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.12.0" +version = "1.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" +checksum = "11493b0bad143270fb8ad284a096dd529ba91924c5409adeac856cc1bf047dbc" dependencies = [ "aws-credential-types", "aws-runtime", @@ -321,14 +333,13 @@ dependencies = [ "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", "fastrand", "hex", - "http 1.5.0", - "sha1 0.10.7", + "http 1.4.2", + "sha1 0.10.6", "time", "tokio", "tracing", @@ -338,9 +349,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.3.0" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" +checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -350,9 +361,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.18.1" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -361,22 +372,21 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.45.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" dependencies = [ "cc", "cmake", "dunce", "fs_extra", - "pkg-config", ] [[package]] name = "aws-runtime" -version = "1.9.2" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef47857a1d4488b528f4a5d5715fa7c3300820897824152234d3fa22b1426657" +checksum = "5fc0651c57e384202e47153c1260b84a9936e19803d747615edf199dc3b98d17" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -389,8 +399,8 @@ dependencies = [ "bytes", "bytes-utils", "fastrand", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "percent-encoding", "pin-project-lite", "tracing", @@ -399,11 +409,10 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.109.0" +version = "1.97.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3cfe74df5d9ad2fedd691973ad3521ebf4f27a3c68c792556686aedb5519bab" +checksum = "9aadc669e184501caaa6beafb28c6267fc1baef0810fb58f9b205485ca3f2567" dependencies = [ - "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", @@ -412,24 +421,22 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", "fastrand", "http 0.2.12", - "http 1.5.0", + "http 1.4.2", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-ssooidc" -version = "1.111.0" +version = "1.99.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81b0ec31ed6191bd11350aae4b2004198f2db21350cb0a20c57e0a92e55dd161" +checksum = "1342a7db8f358d3de0aed2007a0b54e875458e39848d54cc1d46700b2bfcb0a8" dependencies = [ - "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", @@ -438,24 +445,22 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", "fastrand", "http 0.2.12", - "http 1.5.0", + "http 1.4.2", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-sts" -version = "1.114.0" +version = "1.102.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef45745026107ec30c4ef86bd8ae4b002e7e5f6a86e4225240bdf6b06a0b944a" +checksum = "0fc35b7a14cabdad13795fbbbd26d5ddec0882c01492ceedf2af575aad5f37dd" dependencies = [ - "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", @@ -465,22 +470,21 @@ dependencies = [ "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-schema", "aws-smithy-types", "aws-smithy-xml", "aws-types", "fastrand", "http 0.2.12", - "http 1.5.0", + "http 1.4.2", "regex-lite", "tracing", ] [[package]] name = "aws-sigv4" -version = "1.5.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f" +checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -489,20 +493,20 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.12.1", "http 0.2.12", - "http 1.5.0", + "http 1.4.2", "percent-encoding", - "sha2", + "sha2 0.10.9", "time", "tracing", ] [[package]] name = "aws-smithy-async" -version = "1.3.0" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" +checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" dependencies = [ "futures-util", "pin-project-lite", @@ -511,9 +515,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.64.0" +version = "0.63.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" +checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -521,8 +525,8 @@ dependencies = [ "bytes-utils", "futures-core", "futures-util", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "percent-encoding", "pin-project-lite", @@ -532,15 +536,15 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.4.0" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebfd138fac0337cee7516c352757ea73b9f2266e57d0bcb5bc70e9547e45aef1" +checksum = "5c3ef8931ad1c98aa6a55b4256f847f3116090819844e0dd41ea682cac5dd2d3" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", "h2", - "http 1.5.0", + "http 1.4.2", "hyper", "hyper-rustls", "hyper-util", @@ -556,9 +560,9 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.63.0" +version = "0.62.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dc65a121adb4b33729919fcfa14fa36fb33c1555a8f06bb0e2188dbfdc1d9ef" +checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-schema", @@ -567,31 +571,28 @@ dependencies = [ [[package]] name = "aws-smithy-observability" -version = "0.3.0" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" +checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-query" -version = "0.62.0" +version = "0.60.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62" +checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" dependencies = [ - "aws-smithy-runtime-api", - "aws-smithy-schema", "aws-smithy-types", - "aws-smithy-xml", "urlencoding", ] [[package]] name = "aws-smithy-runtime" -version = "1.14.0" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b82e438d30e02a825d363bd639a9efaed68a8089d86101054b0081e7e0d3e606" +checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -603,9 +604,9 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "http 1.5.0", + "http 1.4.2", "http-body 0.4.6", - "http-body 1.1.0", + "http-body 1.0.1", "http-body-util", "pin-project-lite", "pin-utils", @@ -615,16 +616,16 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.16.0" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223" +checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.5.0", + "http 1.4.2", "pin-project-lite", "tokio", "tracing", @@ -633,39 +634,39 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api-macros" -version = "1.1.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" +checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "aws-smithy-schema" -version = "0.2.0" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" +checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", - "http 1.5.0", + "http 1.4.2", ] [[package]] name = "aws-smithy-types" -version = "1.6.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9" +checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85" dependencies = [ "base64-simd", "bytes", "bytes-utils", "http 0.2.12", - "http 1.5.0", + "http 1.4.2", "http-body 0.4.6", - "http-body 1.1.0", + "http-body 1.0.1", "http-body-util", "itoa", "num-integer", @@ -678,21 +679,18 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.62.0" +version = "0.60.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8" +checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" dependencies = [ - "aws-smithy-runtime-api", - "aws-smithy-schema", - "aws-smithy-types", "xmlparser", ] [[package]] name = "aws-types" -version = "1.6.0" +version = "1.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" +checksum = "d16bf10b03a3c01e6b3b7d47cd964e873ffe9e7d4e80fad16bd4c077cb068531" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -714,8 +712,8 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-util", @@ -745,8 +743,8 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -764,7 +762,7 @@ checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -777,8 +775,8 @@ dependencies = [ "bytes", "either", "fs-err", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "hyper", "hyper-util", "pin-project-lite", @@ -806,12 +804,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" - [[package]] name = "base64-simd" version = "0.8.0" @@ -848,21 +840,22 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.13.2" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "blake3" -version = "1.8.7" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" dependencies = [ + "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", ] [[package]] @@ -891,15 +884,15 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytemuck" -version = "1.25.2" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" [[package]] name = "bytes" -version = "1.12.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" dependencies = [ "serde", ] @@ -916,9 +909,9 @@ dependencies = [ [[package]] name = "bytesize" -version = "2.7.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7354288c522e7e980fafd2075d63d1285794c3a6a16cdd492f189ea406e5f18b" +checksum = "351a3e803ee3c6eaeee6b00076b767514b37c32a73d326c3ec7abddb7d6c3493" [[package]] name = "bytestring" @@ -940,9 +933,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.5" +version = "1.2.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" +checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" dependencies = [ "find-msvc-tools", "jobserver", @@ -958,18 +951,18 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" -version = "0.2.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chacha20" -version = "0.10.2" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", "rand_core 0.10.1", ] @@ -1000,9 +993,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.6" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -1010,9 +1003,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.6" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", @@ -1022,14 +1015,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.4" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] @@ -1053,6 +1046,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "cmsketch" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7ee2cfacbd29706479902b06d75ad8f1362900836aa32799eabc7e004bfd854" + [[package]] name = "colorchoice" version = "1.0.5" @@ -1061,9 +1060,9 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "combine" -version = "4.6.8" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" dependencies = [ "bytes", "memchr", @@ -1085,22 +1084,22 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.41" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58a6d0db8759036a783bc7c3f7a07f8cef3bf9470eb1db3bc86e8bcd1c5d0fe8" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" dependencies = [ "compression-core", "flate2", "memchr", - "zstd 0.14.0", - "zstd-safe 8.0.0", + "zstd", + "zstd-safe", ] [[package]] name = "compression-core" -version = "0.4.33" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" [[package]] name = "concurrent-queue" @@ -1167,9 +1166,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" dependencies = [ "libc", ] @@ -1186,18 +1185,18 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-epoch" -version = "0.9.21" +version = "0.9.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" dependencies = [ "crossbeam-utils", ] @@ -1214,9 +1213,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.23" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crypto-common" @@ -1277,7 +1276,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -1288,7 +1287,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -1307,15 +1306,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.11.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" - -[[package]] -name = "datasketches" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "der-parser" @@ -1348,6 +1341,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "crypto-common 0.1.7", + "subtle", ] [[package]] @@ -1364,13 +1358,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.7" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] @@ -1393,7 +1387,7 @@ checksum = "f88959de2d447fd3eddcf1909d1f19fe084e27a056a6904203dc5d8b9e771c1e" dependencies = [ "rust_decimal", "serde", - "thiserror 2.0.20", + "thiserror 2.0.18", "time", "winnow 0.6.26", ] @@ -1406,15 +1400,15 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" -version = "1.18.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" [[package]] name = "enumset" -version = "1.1.14" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccc5801fd11762e24d1e420d01d2ac518f2a2ca4329d4fbb6639f2412b6204e0" +checksum = "839c4174b41e75c8f7306110b2c51996a293b8d1d850edd529011841d9fede7d" dependencies = [ "enumset_derive", ] @@ -1428,7 +1422,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -1444,15 +1438,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] name = "event-listener" -version = "5.4.2" +version = "5.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" dependencies = [ + "concurrent-queue", "parking", "pin-project-lite", ] @@ -1469,11 +1464,10 @@ dependencies = [ [[package]] name = "eyre" -version = "0.6.14" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08309dbcc659c5549a24ddb9b27027640641b282ef5768267c7e675558986a3" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" dependencies = [ - "autocfg", "indenter", "once_cell", ] @@ -1486,7 +1480,7 @@ checksum = "c29b33a0187823f1fa88b36980227dc96c7504ede2288e7d2a77d9d6d88b260c" dependencies = [ "log", "once_cell", - "rand 0.9.5", + "rand 0.9.4", "tokio", ] @@ -1502,9 +1496,21 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.5.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "faststr" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca7d44d22004409a61c393afb3369c8f7bb74abcae49fe249ee01dcc3002113" +dependencies = [ + "bytes", + "rkyv", + "serde", + "simdutf8", +] [[package]] name = "figment" @@ -1524,9 +1530,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.12" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flatbuffers" @@ -1540,13 +1546,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.10" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", "miniz_oxide", - "zlib-rs", ] [[package]] @@ -1572,18 +1577,18 @@ dependencies = [ [[package]] name = "foyer" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a59f42276891c0a4ce683fcda1aa1b4fd1065d68116c264e4bf49b9d318a0ed" +checksum = "3b0abc0b87814989efa711f9becd9f26969820e2d3905db27d10969c4bd45890" dependencies = [ "anyhow", - "asyncband", "equivalent", "foyer-common", "foyer-memory", "foyer-storage", "foyer-tokio", "futures-util", + "mea", "mixtrics", "pin-project", "serde", @@ -1592,9 +1597,9 @@ dependencies = [ [[package]] name = "foyer-common" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643b47d510032a01e5af70dca288b0c5ec531646c1a8392e793fb5b0c13bef30" +checksum = "a3db80d5dece93adb7ad709c84578794724a9cba342a7e566c3551c7ec626789" dependencies = [ "anyhow", "bincode", @@ -1619,21 +1624,21 @@ dependencies = [ [[package]] name = "foyer-memory" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae51f5089f3d9025ae77f17ea66d2489ac0f82fbb1d91462807bea174d486d82" +checksum = "db907f40a527ca2aa2f40a5f68b32ea58aa70f050cd233518e9ffd402cfba6ce" dependencies = [ "anyhow", - "asyncband", "bitflags", - "datasketches", + "cmsketch", "equivalent", "foyer-common", "foyer-intrusive-collections", "foyer-tokio", "futures-util", - "hashbrown 0.17.1", - "itertools 0.15.0", + "hashbrown 0.16.1", + "itertools 0.14.0", + "mea", "mixtrics", "parking_lot", "paste", @@ -1644,13 +1649,12 @@ dependencies = [ [[package]] name = "foyer-storage" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "118192f532ba013f0cdc607efc22324b9ed855baed185608af0daa986e835c6b" +checksum = "1983f1db3d0710e9c9d5fc116d9202dccd41a2d1e032572224f1aff5520aa958" dependencies = [ "allocator-api2", "anyhow", - "asyncband", "bytes", "core_affinity", "equivalent", @@ -1661,34 +1665,35 @@ dependencies = [ "fs4", "futures-core", "futures-util", - "hashbrown 0.17.1", + "hashbrown 0.16.1", "io-uring", - "itertools 0.15.0", + "itertools 0.14.0", "libc", "lz4", + "mea", "parking_lot", "pin-project", - "rand 0.10.2", + "rand 0.9.4", "serde", "tracing", "twox-hash", - "zstd 0.13.3", + "zstd", ] [[package]] name = "foyer-tokio" -version = "0.22.6" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6741d1133dfaab64d3f8a9290bbfed3685084add28f8b6ee7a6264aa4dc26918" +checksum = "f6577b05a7ffad0db555aedf00bfe52af818220fc4c1c3a7a12520896fc38627" dependencies = [ "tokio", ] [[package]] name = "fs-err" -version = "3.3.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91aa448ca50d7e79433bdf3ee8d99215430d2ec02ade5aefab2a073a1822e8a" +checksum = "73fde052dbfc920003cfd2c8e2c6e6d4cc7c1091538c3a24226cec0665ab08c0" dependencies = [ "autocfg", "tokio", @@ -1712,9 +1717,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" dependencies = [ "futures-channel", "futures-core", @@ -1727,9 +1732,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", "futures-sink", @@ -1737,15 +1742,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" [[package]] name = "futures-executor" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" dependencies = [ "futures-core", "futures-task", @@ -1754,38 +1759,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" [[package]] name = "futures-macro" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] name = "futures-sink" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" [[package]] name = "futures-task" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" [[package]] name = "futures-util" -version = "0.3.34" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-channel", "futures-core", @@ -1818,7 +1823,7 @@ dependencies = [ "libc", "wasi", "wasm-bindgen", - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -1828,9 +1833,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 5.3.0", "wasip2", + "wasm-bindgen", ] [[package]] @@ -1840,11 +1847,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", - "wasm-bindgen", ] [[package]] @@ -1854,7 +1859,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" dependencies = [ "polyval", - "zeroize", ] [[package]] @@ -1871,16 +1875,16 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.19" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.5.0", + "http 1.4.2", "indexmap", "slab", "tokio", @@ -1900,6 +1904,17 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -1925,9 +1940,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.5.3" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "hex" @@ -1945,6 +1960,15 @@ dependencies = [ "vsimd", ] +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "hmac" version = "0.13.0" @@ -1967,9 +1991,9 @@ dependencies = [ [[package]] name = "http" -version = "1.5.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes", "itoa", @@ -1988,24 +2012,24 @@ dependencies = [ [[package]] name = "http-body" -version = "1.1.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.5.0", + "http 1.4.2", ] [[package]] name = "http-body-util" -version = "0.1.5" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "pin-project-lite", ] @@ -2029,26 +2053,26 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" -version = "0.4.15" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" dependencies = [ "typenum", ] [[package]] name = "hyper" -version = "1.11.1" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", "h2", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -2064,7 +2088,7 @@ version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.5.0", + "http 1.4.2", "hyper", "hyper-util", "rustls", @@ -2080,12 +2104,12 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64 0.22.1", + "base64", "bytes", "futures-channel", "futures-util", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "hyper", "ipnet", "libc", @@ -2123,9 +2147,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" dependencies = [ "displaydoc", "potential_utf", @@ -2137,9 +2161,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" dependencies = [ "displaydoc", "litemap", @@ -2150,9 +2174,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -2164,17 +2188,16 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" [[package]] name = "icu_properties" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" dependencies = [ - "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -2185,15 +2208,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.3.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" [[package]] name = "icu_provider" -version = "2.3.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" dependencies = [ "displaydoc", "icu_locale_core", @@ -2239,9 +2262,9 @@ checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" [[package]] name = "indexmap" -version = "2.14.2" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -2264,9 +2287,9 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.15" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" +checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" dependencies = [ "bitflags", "cfg-if", @@ -2275,9 +2298,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.2" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" [[package]] name = "is_terminal_polyfill" @@ -2321,7 +2344,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.20", + "thiserror 2.0.18", "walkdir", "windows-link 0.2.1", ] @@ -2336,7 +2359,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn 2.0.119", + "syn", ] [[package]] @@ -2355,24 +2378,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "jobserver" -version = "0.1.35" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ - "getrandom 0.4.3", + "getrandom 0.3.4", "libc", ] [[package]] name = "js-sys" -version = "0.3.105" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" dependencies = [ "cfg-if", "futures-util", @@ -2387,9 +2410,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.189" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libloading" @@ -2409,22 +2432,22 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "linkme" -version = "0.3.37" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" +checksum = "e83272d46373fb8decca684579ac3e7c8f3d71d4cc3aa693df8759e260ae41cf" dependencies = [ "linkme-impl", ] [[package]] name = "linkme-impl" -version = "0.3.37" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" +checksum = "32d59e20403c7d08fe62b4376edfe5c7fb2ef1e6b1465379686d0f21c8df444b" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] @@ -2441,9 +2464,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" [[package]] name = "lock_api" @@ -2456,15 +2479,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.34" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" [[package]] name = "lru" -version = "0.18.4" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" +checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" dependencies = [ "hashbrown 0.17.1", ] @@ -2510,7 +2533,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06138aebc1442e86b201f492616a51690a758d7b75253162f9e9208382ca2bb7" dependencies = [ "libc", - "rand 0.10.2", + "rand 0.10.1", "tokio", "turmoil", ] @@ -2541,10 +2564,19 @@ dependencies = [ ] [[package]] -name = "memchr" -version = "2.8.3" +name = "mea" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +checksum = "2640d335e7273dacdcf51044026139b2e269c3bb0dfc3f8cb3496b85e3f6a42c" +dependencies = [ + "slab", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" [[package]] name = "memoffset" @@ -2569,9 +2601,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.9.1" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", "simd-adler32", @@ -2579,9 +2611,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.3" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -2598,6 +2630,26 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "munge" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" +dependencies = [ + "munge_macro", +] + +[[package]] +name = "munge_macro" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "nix" version = "0.31.3" @@ -2644,14 +2696,14 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] name = "num-bigint" -version = "0.4.8" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", @@ -2665,9 +2717,9 @@ checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" -version = "0.1.47" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ "num-traits", ] @@ -2719,13 +2771,13 @@ dependencies = [ [[package]] name = "object_store" -version = "0.14.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50" +checksum = "765784b4390c6bcf80316e5a22f4e3661b639c9d8c83246856643c27d8ce9dbe" dependencies = [ "async-trait", "aws-lc-rs", - "base64 0.22.1", + "base64", "bytes", "chrono", "crc-fast", @@ -2733,7 +2785,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http 1.5.0", + "http 1.4.2", "http-body-util", "humantime", "hyper", @@ -2742,14 +2794,14 @@ dependencies = [ "nix", "parking_lot", "percent-encoding", - "quick-xml", - "rand 0.10.2", + "quick-xml 0.40.1", + "rand 0.10.1", "reqwest", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.20", + "thiserror 2.0.18", "tokio", "tracing", "url", @@ -2807,7 +2859,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -2871,16 +2923,16 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "pem" -version = "4.0.0" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64 0.23.1", + "base64", "serde_core", ] @@ -2907,7 +2959,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -2924,27 +2976,26 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.34" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "polyval" -version = "0.7.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" +checksum = "7dfc63250416fea14f5749b90725916a6c903f599d51cb635aa7a52bfd03eede" dependencies = [ "cpubits", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", "universal-hash", - "zeroize", ] [[package]] name = "potential_utf" -version = "0.1.6" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" dependencies = [ "zerovec", ] @@ -2966,9 +3017,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.107" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] @@ -2981,7 +3032,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", "version_check", "yansi", ] @@ -3022,7 +3073,7 @@ dependencies = [ "parking_lot", "procfs", "protobuf", - "thiserror 2.0.20", + "thiserror 2.0.18", ] [[package]] @@ -3045,7 +3096,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -3068,6 +3119,36 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "ptr_meta" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" +dependencies = [ + "ptr_meta_derive", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "quick-xml" +version = "0.40.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quick-xml" version = "0.41.0" @@ -3080,9 +3161,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.11" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" dependencies = [ "bytes", "cfg_aliases", @@ -3092,7 +3173,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.20", + "thiserror 2.0.18", "tokio", "tracing", "web-time", @@ -3100,22 +3181,21 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.17" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.4.3", + "getrandom 0.3.4", "lru-slab", - "rand 0.10.2", - "rand_pcg", + "rand 0.9.4", "ring", "rustc-hash", "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.20", + "thiserror 2.0.18", "tinyvec", "tracing", "web-time", @@ -3123,23 +3203,23 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.15" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" dependencies = [ "cfg_aliases", "libc", "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.47" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -3157,10 +3237,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] -name = "rand" -version = "0.8.8" +name = "rancor" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" +dependencies = [ + "ptr_meta", +] + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -3169,9 +3258,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.5" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -3179,9 +3268,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.2" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -3239,16 +3328,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.5", -] - -[[package]] -name = "rand_pcg" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" -dependencies = [ - "rand_core 0.10.1", + "rand 0.9.4", ] [[package]] @@ -3262,9 +3342,9 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.14.10" +version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055" dependencies = [ "aws-lc-rs", "pem", @@ -3285,41 +3365,29 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.27" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.27" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", -] - -[[package]] -name = "regex" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", + "syn", ] [[package]] name = "regex-automata" -version = "0.4.18" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" dependencies = [ "aho-corasick", "memchr", @@ -3339,18 +3407,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] -name = "reqwest" -version = "0.13.5" +name = "rend" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" + +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64 0.23.1", + "base64", "bytes", "futures-core", "futures-util", "h2", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-rustls", @@ -3392,10 +3466,39 @@ dependencies = [ ] [[package]] -name = "rust_decimal" -version = "1.43.0" +name = "rkyv" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7653272e75dcac41dc199fbea6f5797633994fafd339943c06c9af16bf29cd3a" +checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" +dependencies = [ + "bytes", + "hashbrown 0.17.1", + "indexmap", + "munge", + "ptr_meta", + "rancor", + "rend", + "rkyv_derive", + "tinyvec", + "uuid", +] + +[[package]] +name = "rkyv_derive" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "rust_decimal" +version = "1.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be2a24f50780bc85f09cc6ac299bdf1424302742d77221106859c9d8b102126a" dependencies = [ "arrayvec", "num-traits", @@ -3403,9 +3506,9 @@ dependencies = [ [[package]] name = "rustc-hash" -version = "2.1.3" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" [[package]] name = "rustc_version" @@ -3458,14 +3561,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", "log", @@ -3490,9 +3593,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.15.1" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "web-time", "zeroize", @@ -3516,7 +3619,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3527,9 +3630,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", @@ -3539,9 +3642,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.23" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" @@ -3551,7 +3654,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s2-api" -version = "0.31.5" +version = "0.31.1" dependencies = [ "axum", "base64ct", @@ -3560,23 +3663,24 @@ dependencies = [ "flate2", "futures-core", "futures-util", - "http 1.5.0", + "http 1.4.2", "itertools 0.15.0", "mime", "prost", "s2-common", "serde", "serde_json", + "sonic-rs", "strum", - "thiserror 2.0.20", + "thiserror 2.0.18", "time", "tokio-util", - "zstd 0.14.0", + "zstd", ] [[package]] name = "s2-common" -version = "0.41.3" +version = "0.41.1" dependencies = [ "axum", "base64ct", @@ -3584,18 +3688,18 @@ dependencies = [ "clap", "compact_str", "enumset", - "http 1.5.0", - "rand 0.10.2", + "http 1.4.2", + "rand 0.10.1", "secrecy", "serde", "strum", - "thiserror 2.0.20", + "thiserror 2.0.18", "time", ] [[package]] name = "s2-lite" -version = "0.42.13" +version = "0.42.7" dependencies = [ "async-stream", "async-trait", @@ -3609,13 +3713,13 @@ dependencies = [ "dashmap", "eyre", "futures", - "http 1.5.0", + "http 1.4.2", "indexmap", "itertools 0.15.0", "parking_lot", "prometheus", "prost", - "rand 0.10.2", + "rand 0.10.1", "rcgen", "rustls", "s2-api", @@ -3626,12 +3730,12 @@ dependencies = [ "serde_json", "slatedb", "strum", - "thiserror 2.0.20", + "thiserror 2.0.18", "tikv-jemallocator", "time", "tokio", "tokio-util", - "tower-http 0.7.1", + "tower-http 0.7.0", "tracing", "tracing-subscriber", ] @@ -3640,7 +3744,6 @@ dependencies = [ name = "s2-resource-spec" version = "0.2.2" dependencies = [ - "compact_str", "humantime", "s2-common", "schemars", @@ -3650,25 +3753,23 @@ dependencies = [ [[package]] name = "s2-sdk" -version = "0.34.9" +version = "0.34.3" dependencies = [ "async-compression", "async-stream", "async-trait", "bytes", - "compact_str", "futures-core", "futures-util", - "h2", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "hyper", "hyper-rustls", "hyper-util", "pin-project-lite", "prost", - "rand 0.10.2", + "rand 0.10.1", "rustls", "s2-api", "s2-common", @@ -3676,7 +3777,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.20", + "thiserror 2.0.18", "time", "tokio", "tokio-muxt", @@ -3701,12 +3802,12 @@ dependencies = [ "fastrand", "futures", "getrandom 0.2.15", - "http 1.5.0", + "http 1.4.2", "http-body-util", "hyper", "hyper-util", "mad-turmoil", - "rand 0.10.2", + "rand 0.10.1", "s2-lite", "s2-sdk", "s2-verification", @@ -3722,17 +3823,17 @@ dependencies = [ [[package]] name = "s2-storage" -version = "0.2.5" +version = "0.2.4" dependencies = [ "aegis", "aes-gcm", "blake3", "bytes", - "rand 0.10.2", + "rand 0.10.1", "s2-common", "secrecy", "serde", - "thiserror 2.0.20", + "thiserror 2.0.18", ] [[package]] @@ -3744,7 +3845,7 @@ dependencies = [ "clap", "eyre", "futures", - "rand 0.8.8", + "rand 0.8.6", "s2-sdk", "serde", "serde_json", @@ -3757,9 +3858,9 @@ dependencies = [ [[package]] name = "s3s" -version = "0.15.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "652451a66fefda01a13dc7df24aa2af9e70c7058f98bc08fe2f7c552eaa9f1e3" +checksum = "abe1bd31748cb69848c2cf4028cecdadf5f8299ef3b02a83e21b20e7bda3aba9" dependencies = [ "arc-swap", "arrayvec", @@ -3773,9 +3874,9 @@ dependencies = [ "crc-fast", "futures", "hex-simd", - "hmac", - "http 1.5.0", - "http-body 1.1.0", + "hmac 0.13.0", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "httparse", "hyper", @@ -3786,18 +3887,17 @@ dependencies = [ "nom 8.0.0", "numeric_cast", "pin-project-lite", - "quick-xml", - "regex", + "quick-xml 0.41.0", "serde", "serde_json", "serde_urlencoded", "sha1 0.11.0", - "sha2", + "sha2 0.11.0", "smallvec", "std-next", "subtle", "sync_wrapper", - "thiserror 2.0.20", + "thiserror 2.0.18", "time", "tokio", "tower", @@ -3805,7 +3905,6 @@ dependencies = [ "transform-stream", "url", "urlencoding", - "xxhash-rust", "zeroize", ] @@ -3829,9 +3928,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.2" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" dependencies = [ "dyn-clone", "ref-cast", @@ -3842,14 +3941,14 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "1.2.2" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" +checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 3.0.5", + "syn", ] [[package]] @@ -3904,9 +4003,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.229" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", "serde_derive", @@ -3914,40 +4013,40 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.229" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.229" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] name = "serde_derive_internals" -version = "0.30.0" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -4003,9 +4102,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.7" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", @@ -4019,10 +4118,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", "digest 0.11.3", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha2" version = "0.11.0" @@ -4030,7 +4140,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", + "cpufeatures 0.3.0", "digest 0.11.3", ] @@ -4061,15 +4171,15 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.10" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" [[package]] name = "simd_cesu8" -version = "1.2.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" dependencies = [ "rustc_version", "simdutf8", @@ -4095,9 +4205,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "slatedb" -version = "0.16.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb40332f41e231926df3a0ef742c05316b43368ba4b520433d2a1eba1ab00f0f" +checksum = "35ca56b01922b15aa69fe3abb62cadc985d86032c9647e4606e211c4da751a76" dependencies = [ "async-channel", "async-trait", @@ -4121,7 +4231,7 @@ dependencies = [ "object_store", "ouroboros", "parking_lot", - "rand 0.9.5", + "rand 0.9.4", "serde", "serde_json", "siphasher", @@ -4137,19 +4247,19 @@ dependencies = [ "url", "uuid", "walkdir", - "zstd 0.13.3", + "zstd", ] [[package]] name = "slatedb-common" -version = "0.16.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b77f238f348e95e1653a5235f880ac703011b82582bd556f2aba405abf8180e" +checksum = "0aa8de522ff46a0f9b5a66f45e650d125421d4d91990933591092f9d010c40d1" dependencies = [ "chrono", "log", "object_store", - "rand 0.9.5", + "rand 0.9.4", "rand_xoshiro", "serde", "thread_local", @@ -4158,9 +4268,9 @@ dependencies = [ [[package]] name = "slatedb-txn-obj" -version = "0.16.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad63b6c250219e26d71f497820f970274c301f896366daa56915f2a05df0e0a" +checksum = "d85fd9c0c86dd4954524fa8238d0548bd80f4a9a66983cbde3728402d339993e" dependencies = [ "async-trait", "bytes", @@ -4181,18 +4291,18 @@ checksum = "88414a5ca1f85d82cc34471e975f0f74f6aa54c40f062efa42c0080e7f763f81" [[package]] name = "smallvec" -version = "1.16.0" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] name = "socket2" -version = "0.6.5" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -4202,10 +4312,49 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45e14297decde697ddf377c25752aead0927d5cfc89c2684d2af96901a4ceeea" [[package]] -name = "spin" -version = "0.10.1" +name = "sonic-number" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" +checksum = "3775c3390edf958191f1ab1e8c5c188907feebd0f3ce1604cb621f72961dbf32" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "sonic-rs" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d971cc77a245ccf1756dbd1a87c3e7f709c0191464096510d43eec056d0f2c4f" +dependencies = [ + "ahash", + "bumpalo", + "bytes", + "cfg-if", + "faststr", + "itoa", + "ref-cast", + "serde", + "simdutf8", + "sonic-number", + "sonic-simd", + "thiserror 2.0.18", + "zmij", +] + +[[package]] +name = "sonic-simd" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f99e664ecd2d85a68c87e3c7a3cfe691f647ea9e835de984aba4d54a41f817d4" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" [[package]] name = "stable_deref_trait" @@ -4226,7 +4375,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04082e93ed1a06debd9148c928234b46d2cf260bc65f44e1d1d3fa594c5beebc" dependencies = [ "simdutf8", - "thiserror 2.0.20", + "thiserror 2.0.18", ] [[package]] @@ -4253,7 +4402,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4264,20 +4413,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.119" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "3.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -4301,7 +4439,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4329,11 +4467,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.20" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl 2.0.20", + "thiserror-impl 2.0.18", ] [[package]] @@ -4344,25 +4482,25 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "thiserror-impl" -version = "2.0.20" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] name = "thread_local" -version = "1.1.10" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", ] @@ -4389,9 +4527,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.55" +version = "0.3.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" dependencies = [ "deranged", "num-conv", @@ -4409,9 +4547,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.32" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" dependencies = [ "num-conv", "time-core", @@ -4419,9 +4557,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.4" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" dependencies = [ "displaydoc", "zerovec", @@ -4429,9 +4567,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.13.2" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -4461,13 +4599,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.2" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] @@ -4482,9 +4620,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.5" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ "rustls", "tokio", @@ -4492,9 +4630,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.19" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" dependencies = [ "futures-core", "pin-project-lite", @@ -4503,16 +4641,15 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", "futures-sink", "futures-util", "hashbrown 0.15.5", - "libc", "pin-project-lite", "tokio", ] @@ -4583,8 +4720,8 @@ dependencies = [ "bitflags", "bytes", "futures-util", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "pin-project-lite", "tower", "tower-layer", @@ -4594,16 +4731,16 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.7.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c" +checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" dependencies = [ "async-compression", "bitflags", "bytes", "futures-core", - "http 1.5.0", - "http-body 1.1.0", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "percent-encoding", "pin-project-lite", @@ -4646,7 +4783,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -4724,7 +4861,7 @@ checksum = "0061c28f1469e94771bb8aa626ce6b29265c2fb5034115046a170b0cba2e33d2" dependencies = [ "bytes", "indexmap", - "rand 0.9.5", + "rand 0.9.4", "rand_distr", "scoped-tls", "tokio", @@ -4733,9 +4870,12 @@ dependencies = [ [[package]] name = "twox-hash" -version = "2.1.4" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +dependencies = [ + "rand 0.9.4", +] [[package]] name = "typenum" @@ -4749,7 +4889,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" dependencies = [ - "rand 0.9.5", + "rand 0.9.4", "serde", "web-time", ] @@ -4829,13 +4969,13 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.26.1" +version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" +checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ "getrandom 0.4.3", "js-sys", - "rand 0.10.2", + "rand 0.10.1", "serde_core", "wasm-bindgen", ] @@ -4894,9 +5034,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.128" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" dependencies = [ "cfg-if", "once_cell", @@ -4907,9 +5047,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.78" +version = "0.4.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" dependencies = [ "js-sys", "wasm-bindgen", @@ -4917,9 +5057,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.128" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4927,22 +5067,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.128" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 3.0.5", + "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.128" +version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" dependencies = [ "unicode-ident", ] @@ -4962,9 +5102,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.105" +version = "0.3.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" dependencies = [ "js-sys", "wasm-bindgen", @@ -4982,9 +5122,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" dependencies = [ "rustls-pki-types", ] @@ -5011,7 +5151,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5087,7 +5227,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -5098,7 +5238,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -5165,7 +5305,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -5174,7 +5314,16 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", ] [[package]] @@ -5192,14 +5341,31 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link 0.2.1", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -5217,48 +5383,96 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "0.6.26" @@ -5285,9 +5499,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.4" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "x509-parser" @@ -5303,7 +5517,7 @@ dependencies = [ "nom 7.1.3", "oid-registry", "rusticata-macros", - "thiserror 2.0.20", + "thiserror 2.0.18", "time", ] @@ -5315,9 +5529,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" -version = "0.8.18" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" [[package]] name = "yansi" @@ -5354,28 +5568,28 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.57" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.57" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] @@ -5395,7 +5609,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", "synstructure", ] @@ -5407,9 +5621,9 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.5" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" dependencies = [ "displaydoc", "yoke", @@ -5418,9 +5632,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.8" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" dependencies = [ "yoke", "zerofrom", @@ -5429,26 +5643,20 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.6" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] -[[package]] -name = "zlib-rs" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" - [[package]] name = "zmij" -version = "1.0.23" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" [[package]] name = "zstd" @@ -5456,41 +5664,23 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe 7.3.0", -] - -[[package]] -name = "zstd" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e" -dependencies = [ - "zstd-safe 8.0.0", + "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.3.0" +version = "7.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-safe" -version = "8.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.1.0+zstd.1.5.7" +version = "2.0.16+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" dependencies = [ "cc", "pkg-config", diff --git a/sim/Cargo.toml b/sim/Cargo.toml index 1293bd1e..2b537aad 100644 --- a/sim/Cargo.toml +++ b/sim/Cargo.toml @@ -19,17 +19,17 @@ path = "src/main.rs" async-trait = "0.1" axum = "0.8" blake3 = "1.8" -bytes = "1.12" -bytesize = "2.7" +bytes = "1.11" +bytesize = "2.3" clap = { version = "4.6", features = ["derive", "env"] } eyre = "0.6" fastrand = "2" futures = "0.3" # Activates determinism on Linux in the patched getrandom (see [patch.crates-io]). getrandom = { version = "0.2", features = ["deterministic-simulation"] } -http = "1.5" +http = "1.4" http-body-util = "0.1" -hyper = { version = "1.11", features = ["client", "server", "http1", "http2"] } +hyper = { version = "1.9", features = ["client", "server", "http1", "http2"] } hyper-util = { version = "0.1", features = [ "client-legacy", "http1", @@ -46,10 +46,10 @@ s2-sdk = { path = "../sdk", features = ["_hidden"] } # The Go checker built in CI must be pinned to the same rev (see the # S2_VERIFICATION_REV env in .github/workflows/ci.yml). s2-verification = { git = "https://github.com/s2-streamstore/s2-verification", rev = "b4af8c8ef4965d9b335101c422eadb33f3169004" } -s3s = "0.15.0" +s3s = "0.14.1" serde_json = "1.0" -slatedb = { version = "0.16.0", features = ["lz4", "zstd"] } -tokio = { version = "1.53", features = ["macros", "rt", "sync", "time", "io-util"] } +slatedb = { version = "0.15.0", features = ["lz4", "zstd"] } +tokio = { version = "1.52", features = ["macros", "rt", "sync", "time", "io-util"] } tower = "0.5" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/storage/CHANGELOG.md b/storage/CHANGELOG.md index aeb91386..3719b3b7 100644 --- a/storage/CHANGELOG.md +++ b/storage/CHANGELOG.md @@ -2,22 +2,6 @@ All notable changes to this project will be documented in this file. -## [0.2.6] - 2026-09-25 - -### Miscellaneous Tasks - -- Updated the following local packages: s2-common - - - -## [0.2.5] - 2026-09-22 - -### Miscellaneous Tasks - -- Update Cargo.toml dependencies - - - ## [0.2.4] - 2026-07-22 ### Miscellaneous Tasks diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 1fccbb72..4bd31772 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-storage" -version = "0.2.6" +version = "0.2.4" description = "Storage-layer internals shared by S2 server implementations" edition.workspace = true license.workspace = true diff --git a/testcontainers/CHANGELOG.md b/testcontainers/CHANGELOG.md index 6d8b7d6d..5ed24846 100644 --- a/testcontainers/CHANGELOG.md +++ b/testcontainers/CHANGELOG.md @@ -2,42 +2,6 @@ All notable changes to this project will be documented in this file. -## [0.43.0] - 2026-09-25 - - - -## [0.42.14] - 2026-09-24 - - - -## [0.42.13] - 2026-09-22 - -### Miscellaneous Tasks - -- Update Cargo.toml dependencies - - - -## [0.42.12] - 2026-09-16 - - - -## [0.42.11] - 2026-09-11 - - - -## [0.42.10] - 2026-09-11 - - - -## [0.42.9] - 2026-09-10 - - - -## [0.42.8] - 2026-09-01 - - - ## [0.42.7] - 2026-08-18 diff --git a/testcontainers/Cargo.toml b/testcontainers/Cargo.toml index 23647de1..af38efd8 100644 --- a/testcontainers/Cargo.toml +++ b/testcontainers/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-testcontainers" -version = "0.43.0" +version = "0.42.7" description = "Testcontainers helpers for the S2 Docker image" edition.workspace = true license.workspace = true