diff --git a/.github/actions/rust-dependency-cooldown/.gitignore b/.github/actions/rust-dependency-cooldown/.gitignore new file mode 100644 index 00000000..c18dd8d8 --- /dev/null +++ b/.github/actions/rust-dependency-cooldown/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/.github/actions/rust-dependency-cooldown/README.md b/.github/actions/rust-dependency-cooldown/README.md new file mode 100644 index 00000000..c36831f3 --- /dev/null +++ b/.github/actions/rust-dependency-cooldown/README.md @@ -0,0 +1,51 @@ +# Rust dependency cooldown + +This action checks newly locked crates.io versions against the caller's +`registry.global-min-publish-age` setting. S2-owned crates listed in +`first-party-crates.txt` are exempt from the publication-age check. + +## Security exceptions + +`security-exceptions.toml` records reviewed exceptions for individual security +releases. Each entry requires an exact crate name and version, an advisory +identifier or URL, and a reason: + +```toml +[[exception]] +crate = "rustls" +version = "0.23.45" +advisory = "RUSTSEC-2026-0285" +reason = "Fixes TLS 1.3 handshake messages accepted at the wrong encryption level" +``` + +Review the advisory and confirm that the exact release fixes it before adding an +entry. Ranges, wildcards, duplicate entries, and malformed configuration are +rejected. The action prints the crate, version, advisory, and reason whenever it +uses an exception. The exception waives only publication age; other dependency +checks, including `cargo deny`, continue to apply. + +The exception file is distributed with this action. After merging an exception, +update consuming repositories' pinned action or reusable-workflow commit to pick +it up. Future releases of the same crate still have to satisfy the cooldown. + +Entries need no expiry field: an approved release follows the normal age check +once it is old enough. Old entries may be removed in a later cleanup. An empty +file or `exception = []` means there are no security exceptions. + +Cargo also enforces publication age during dependency resolution. To select an +approved release, override that resolver check for the targeted update command: + +```sh +CARGO_RESOLVER_INCOMPATIBLE_PUBLISH_AGE=allow cargo +nightly update -p rustls --precise 0.23.45 +``` + +Run the cooldown action on the resulting lockfile to check every newly selected +version against the exception list and normal age requirement. + +## Tests + +Run from the repository root: + +```sh +python3 -m unittest discover -s .github/actions/rust-dependency-cooldown -p 'test_*.py' -v +``` diff --git a/.github/actions/rust-dependency-cooldown/check.py b/.github/actions/rust-dependency-cooldown/check.py index 535e2884..53f7d0b0 100644 --- a/.github/actions/rust-dependency-cooldown/check.py +++ b/.github/actions/rust-dependency-cooldown/check.py @@ -20,6 +20,7 @@ ACTION_ROOT = Path(__file__).resolve().parent REPO_ROOT = Path.cwd() CONFIG = REPO_ROOT / ".cargo" / "config.toml" ALLOWLIST = ACTION_ROOT / "first-party-crates.txt" +SECURITY_EXCEPTIONS = ACTION_ROOT / "security-exceptions.toml" CRATES_IO_SOURCE = "registry+https://github.com/rust-lang/crates.io-index" INDEX_BASE = "https://index.crates.io" USER_AGENT = "s2 minimum-publish-age check (github.com/s2-streamstore/s2)" @@ -69,6 +70,36 @@ def allowed_crates() -> set[str]: } +def security_exceptions() -> dict[tuple[str, str], dict[str, str]]: + with SECURITY_EXCEPTIONS.open("rb") as exceptions_file: + document = tomllib.load(exceptions_file) + if document.keys() - {"exception"}: + raise ValueError(f"unexpected fields in {SECURITY_EXCEPTIONS}") + entries = document.get("exception", []) + if not isinstance(entries, list): + raise ValueError(f"expected [[exception]] entries in {SECURITY_EXCEPTIONS}") + required_fields = {"crate", "version", "advisory", "reason"} + exceptions: dict[tuple[str, str], dict[str, str]] = {} + for index, entry in enumerate(entries, start=1): + if not isinstance(entry, dict) or entry.keys() != required_fields: + raise ValueError(f"security exception {index} must contain {sorted(required_fields)}") + for field, value in entry.items(): + if not isinstance(value, str) or not value or value != value.strip(): + raise ValueError(f"security exception {index} has invalid {field}") + if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]*", entry["crate"]) is None: + raise ValueError(f"security exception {index} requires an exact crate name") + if re.fullmatch( + r"[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?", + entry["version"], + ) is None: + raise ValueError(f"security exception {index} requires an exact version") + key = (entry["crate"], entry["version"]) + if key in exceptions: + raise ValueError(f"duplicate security exception for {key[0]} {key[1]}") + exceptions[key] = entry + return exceptions + + def crates_io_versions(lock_text: str) -> set[tuple[str, str]]: packages = tomllib.loads(lock_text).get("package", []) return { @@ -196,6 +227,7 @@ def main() -> int: try: if args.base_ref and run_git("rev-parse", "--verify", "--quiet", args.base_ref).returncode: raise ValueError(f"base ref {args.base_ref!r} is not available") + approved_exceptions = security_exceptions() if args.base_ref and not has_relevant_changes(args.base_ref): print("No Rust dependency cooldown files changed; check skipped.") return 0 @@ -231,6 +263,12 @@ def main() -> int: checked += 1 crate_age = now - pubtime if crate_age < age_limit: + if exception := approved_exceptions.get((name, version)): + print( + f"Publication cooldown waived for {name} {version} ({lockfile}): " + f"{exception['advisory']} — {exception['reason']}" + ) + continue violations.append( f"{name} {version} ({lockfile}): published " f"{crate_age.total_seconds() / 86400:.1f} days ago; " @@ -246,7 +284,7 @@ def main() -> int: print(f" - {violation}", file=sys.stderr) return 1 - print(f"Checked {checked} new crates.io version(s); all are at least {age_text} old.") + print(f"Checked {checked} new crates.io version(s); publication cooldown policy satisfied.") return 0 diff --git a/.github/actions/rust-dependency-cooldown/security-exceptions.toml b/.github/actions/rust-dependency-cooldown/security-exceptions.toml new file mode 100644 index 00000000..c0e3a044 --- /dev/null +++ b/.github/actions/rust-dependency-cooldown/security-exceptions.toml @@ -0,0 +1,5 @@ +[[exception]] +crate = "rustls" +version = "0.23.45" +advisory = "RUSTSEC-2026-0285" +reason = "Fixes TLS 1.3 handshake messages accepted at the wrong encryption level" diff --git a/.github/actions/rust-dependency-cooldown/test_check.py b/.github/actions/rust-dependency-cooldown/test_check.py new file mode 100644 index 00000000..313e59a5 --- /dev/null +++ b/.github/actions/rust-dependency-cooldown/test_check.py @@ -0,0 +1,290 @@ +from __future__ import annotations + +import importlib.util +import io +import json +import subprocess +import sys +import tempfile +import unittest +from contextlib import ExitStack, redirect_stderr, redirect_stdout +from datetime import datetime, timedelta, timezone +from pathlib import Path +from unittest.mock import patch + + +SPEC = importlib.util.spec_from_file_location( + "cooldown_check", Path(__file__).with_name("check.py") +) +assert SPEC is not None and SPEC.loader is not None +check = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(check) + +NOW = datetime(2026, 9, 16, tzinfo=timezone.utc) +APPROVED_EXCEPTION = { + "crate": "rustls", + "version": "0.23.45", + "advisory": "https://rustsec.org/advisories/RUSTSEC-2026-0285.html", + "reason": "First release fixing TLS 1.3 handshake encryption-level validation.", +} + + +def exception_document(*entries: dict[str, object]) -> str: + return "\n".join( + "[[exception]]\n" + + "\n".join(f"{name} = {json.dumps(value)}" for name, value in entry.items()) + + "\n" + for entry in entries + ) + + +class CooldownSecurityExceptionTests(unittest.TestCase): + def setUp(self) -> None: + temporary_directory = tempfile.TemporaryDirectory() + self.addCleanup(temporary_directory.cleanup) + temporary_root = Path(temporary_directory.name) + self.repo = temporary_root / "consumer" + self.repo.mkdir() + self.action = temporary_root / "trusted-action" + self.action.mkdir() + self.exceptions = self.action / "security-exceptions.toml" + self.exceptions.write_text(exception_document(APPROVED_EXCEPTION)) + (self.action / "first-party-crates.txt").write_text("s2-api\n") + (self.repo / ".cargo").mkdir() + (self.repo / ".cargo" / "config.toml").write_text( + '[registry]\nglobal-min-publish-age = "7 days"\n' + ) + self.write_lock() + self.git("init", "--quiet") + self.commit() + + def git(self, *arguments: str) -> None: + subprocess.run( + [ + "git", + "-c", + "user.name=Cooldown Tests", + "-c", + "user.email=cooldown-tests@example.invalid", + "-c", + "commit.gpgsign=false", + "-c", + "core.hooksPath=/dev/null", + *arguments, + ], + cwd=self.repo, + check=True, + capture_output=True, + text=True, + ) + + def commit(self) -> None: + self.git("add", ".") + self.git("commit", "--quiet", "-m", "test: record baseline") + + def write_lock(self, *packages: tuple[str, str, str]) -> None: + contents = "version = 3\n" + for name, version, source in packages: + contents += ( + "\n[[package]]\n" + f"name = {json.dumps(name)}\n" + f"version = {json.dumps(version)}\n" + f"source = {json.dumps(source)}\n" + ) + (self.repo / "Cargo.lock").write_text(contents) + + def run_gate( + self, + publication_times: dict[str, dict[str, datetime]] | None = None, + ) -> tuple[int, str, str, list[str]]: + published = publication_times or { + "rustls": {"0.23.45": NOW - timedelta(days=1)}, + } + output, errors = io.StringIO(), io.StringIO() + with ExitStack() as stack: + for name, value in { + "ACTION_ROOT": self.action, + "REPO_ROOT": self.repo, + "CONFIG": self.repo / ".cargo" / "config.toml", + "ALLOWLIST": self.action / "first-party-crates.txt", + "SECURITY_EXCEPTIONS": self.exceptions, + }.items(): + stack.enter_context(patch.object(check, name, value)) + stack.enter_context( + patch.object(sys, "argv", ["check.py", "--repo-root", str(self.repo), "HEAD"]) + ) + clock = stack.enter_context(patch.object(check, "datetime", wraps=datetime)) + clock.now.return_value = NOW + lookup = stack.enter_context( + patch.object(check, "publication_times", side_effect=published.__getitem__) + ) + stack.enter_context(redirect_stdout(output)) + stack.enter_context(redirect_stderr(errors)) + status = check.main() + lookups = [call.args[0] for call in lookup.call_args_list] + return status, output.getvalue(), errors.getvalue(), lookups + + def test_exact_security_exception_passes_and_reports_justification(self) -> None: + self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) + status, output, errors, _ = self.run_gate() + self.assertEqual(status, 0, errors) + for field in APPROVED_EXCEPTION.values(): + self.assertIn(field, output) + + def test_exception_does_not_cover_another_version_or_crate(self) -> None: + for name, version in [("rustls", "0.23.46"), ("another-crate", "0.23.45")]: + with self.subTest(crate=name, version=version): + self.write_lock((name, version, check.CRATES_IO_SOURCE)) + status, _, errors, _ = self.run_gate( + {name: {version: NOW - timedelta(days=1)}} + ) + self.assertEqual(status, 1) + self.assertIn(f"{name} {version}", errors) + + def test_exception_does_not_hide_an_unrelated_young_dependency(self) -> None: + self.write_lock( + ("rustls", "0.23.45", check.CRATES_IO_SOURCE), + ("another-crate", "1.0.0", check.CRATES_IO_SOURCE), + ) + status, output, errors, _ = self.run_gate( + { + "rustls": {"0.23.45": NOW - timedelta(days=1)}, + "another-crate": {"1.0.0": NOW - timedelta(days=1)}, + } + ) + self.assertEqual(status, 1) + self.assertIn(APPROVED_EXCEPTION["advisory"], output) + self.assertIn("another-crate 1.0.0", errors) + self.assertNotIn("rustls 0.23.45", errors) + + def test_consumer_repository_cannot_supply_its_own_exception(self) -> None: + unapproved = {**APPROVED_EXCEPTION, "version": "0.23.46"} + consumer_action = self.repo / ".github" / "actions" / "rust-dependency-cooldown" + consumer_action.mkdir(parents=True) + for directory in [self.repo, self.repo / ".cargo", consumer_action]: + (directory / "security-exceptions.toml").write_text(exception_document(unapproved)) + self.write_lock(("rustls", "0.23.46", check.CRATES_IO_SOURCE)) + status, _, errors, _ = self.run_gate( + {"rustls": {"0.23.46": NOW - timedelta(days=1)}} + ) + self.assertEqual(status, 1) + self.assertIn("rustls 0.23.46", errors) + + def test_malformed_security_exceptions_fail_closed(self) -> None: + documents = { + "invalid TOML": "[[exception]", + "not an array": "exception = 1\n", + "not a table": "exception = [1]\n", + "unknown top-level field": "exceptions = []\n", + } + for field in APPROVED_EXCEPTION: + missing = {name: value for name, value in APPROVED_EXCEPTION.items() if name != field} + documents[f"missing {field}"] = exception_document(missing) + for value in ["", " ", 123, f" {APPROVED_EXCEPTION[field]}"]: + documents[f"invalid {field}: {value!r}"] = exception_document( + {**APPROVED_EXCEPTION, field: value} + ) + documents["unknown field"] = exception_document({**APPROVED_EXCEPTION, "extra": "value"}) + documents["duplicate pair"] = exception_document(APPROVED_EXCEPTION, APPROVED_EXCEPTION) + self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) + for description, document in documents.items(): + with self.subTest(description=description): + self.exceptions.write_text(document) + status, _, errors, _ = self.run_gate() + self.assertEqual(status, 2) + self.assertIn("minimum-publish-age error", errors) + + def test_missing_action_exception_file_fails_closed(self) -> None: + self.exceptions.unlink() + self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) + status, _, errors, _ = self.run_gate() + self.assertEqual(status, 2) + self.assertIn("minimum-publish-age error", errors) + + def test_malformed_exceptions_fail_even_when_no_lockfile_changed(self) -> None: + self.exceptions.write_text("[[exception]") + status, _, errors, _ = self.run_gate() + self.assertEqual(status, 2) + self.assertIn("minimum-publish-age error", errors) + + def test_empty_exception_lists_do_not_exempt_young_dependencies(self) -> None: + self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) + for document in ["", "exception = []\n"]: + with self.subTest(document=document): + self.exceptions.write_text(document) + status, _, errors, _ = self.run_gate() + self.assertEqual(status, 1) + self.assertIn("rustls 0.23.45", errors) + + def test_exception_ranges_and_wildcards_are_rejected(self) -> None: + self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) + for field, value in [ + ("crate", "rustls*"), + ("version", "*"), + ("version", "0.23.*"), + ("version", ">=0.23.45"), + ]: + with self.subTest(field=field, value=value): + self.exceptions.write_text( + exception_document({**APPROVED_EXCEPTION, field: value}) + ) + status, _, errors, _ = self.run_gate() + self.assertEqual(status, 2) + self.assertIn("minimum-publish-age error", errors) + + def test_approved_version_still_requires_a_publication_time(self) -> None: + self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) + status, _, errors, _ = self.run_gate({"rustls": {}}) + self.assertEqual(status, 2) + self.assertIn("no publication time for rustls 0.23.45", errors) + + def test_mature_approved_version_uses_normal_age_check(self) -> None: + self.write_lock(("rustls", "0.23.45", check.CRATES_IO_SOURCE)) + status, output, errors, lookups = self.run_gate( + {"rustls": {"0.23.45": NOW - timedelta(days=7)}} + ) + self.assertEqual(status, 0, errors) + self.assertNotIn(APPROVED_EXCEPTION["advisory"], output) + self.assertEqual(lookups, ["rustls"]) + + def test_age_boundary_for_ordinary_dependencies_is_unchanged(self) -> None: + self.write_lock(("another-crate", "1.0.0", check.CRATES_IO_SOURCE)) + for age, expected_status in [(timedelta(days=7), 0), (timedelta(days=7, seconds=-1), 1)]: + with self.subTest(age=age): + status, _, errors, _ = self.run_gate({"another-crate": {"1.0.0": NOW - age}}) + self.assertEqual(status, expected_status, errors) + + def test_first_party_allowlist_still_exempts_new_versions(self) -> None: + self.write_lock(("s2-api", "99.0.0", check.CRATES_IO_SOURCE)) + status, _, errors, lookups = self.run_gate() + self.assertEqual(status, 0, errors) + self.assertEqual(lookups, []) + + def test_other_sources_are_not_processed_as_security_exceptions(self) -> None: + for source in [ + "registry+https://example.invalid/index", + "git+https://example.invalid/rustls", + ]: + with self.subTest(source=source): + self.write_lock(("rustls", "0.23.45", source)) + status, output, errors, lookups = self.run_gate() + self.assertEqual(status, 0, errors) + self.assertNotIn(APPROVED_EXCEPTION["advisory"], output) + self.assertEqual(lookups, []) + + def test_existing_locked_versions_are_not_rechecked(self) -> None: + self.write_lock(("already-locked", "1.0.0", check.CRATES_IO_SOURCE)) + self.commit() + self.write_lock( + ("already-locked", "1.0.0", check.CRATES_IO_SOURCE), + ("another-crate", "1.0.0", check.CRATES_IO_SOURCE), + ) + status, _, errors, lookups = self.run_gate( + {"another-crate": {"1.0.0": NOW - timedelta(days=8)}} + ) + self.assertEqual(status, 0, errors) + self.assertEqual(lookups, ["another-crate"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c8a043e..2da7b71d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -310,6 +310,18 @@ jobs: exit 1 fi + - name: Test with persistent volume and WAL storage + run: | + helm template test-release charts/s2-lite-helm \ + --set persistentVolume.enabled=true \ + --set walStorage.persistentVolume.enabled=true \ + --dry-run > /dev/null + helm template test-release charts/s2-lite-helm \ + --set objectStorage.enabled=true \ + --set objectStorage.bucket=test-bucket \ + --set walStorage.bucket=test-wal-bucket \ + --dry-run > /dev/null + build-server: name: Build s2-lite needs: rust-dependency-cooldown diff --git a/.github/workflows/rust-dependency-cooldown-tests.yml b/.github/workflows/rust-dependency-cooldown-tests.yml new file mode 100644 index 00000000..76f56ecc --- /dev/null +++ b/.github/workflows/rust-dependency-cooldown-tests.yml @@ -0,0 +1,22 @@ +name: Rust dependency cooldown tests + +on: + pull_request: + paths: + - .github/actions/rust-dependency-cooldown/** + - .github/workflows/rust-dependency-cooldown-tests.yml + push: + branches: [main] + paths: + - .github/actions/rust-dependency-cooldown/** + - .github/workflows/rust-dependency-cooldown-tests.yml + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - run: python3 -m unittest discover -s .github/actions/rust-dependency-cooldown -p 'test_*.py' -v diff --git a/Cargo.lock b/Cargo.lock index 9fa1927f..de3da95d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,7 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ "crypto-common 0.2.2", - "inout 0.2.2", + "inout", ] [[package]] @@ -39,46 +39,36 @@ dependencies = [ [[package]] name = "aes" -version = "0.8.4" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" dependencies = [ - "cfg-if", - "cipher 0.4.4", - "cpufeatures 0.2.17", -] - -[[package]] -name = "aes" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" -dependencies = [ - "cipher 0.5.2", + "cipher", "cpubits", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", + "zeroize", ] [[package]] name = "aes-gcm" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" +checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f" dependencies = [ "aead", - "aes 0.9.2", - "cipher 0.5.2", + "aes", + "cipher", "ctr", + "ctutils", "ghash", - "subtle", "zeroize", ] [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -97,9 +87,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" dependencies = [ "libc", ] @@ -162,9 +152,9 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "apple-native-keyring-store" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "797f94b6a53d7d10b56dc18290e0d40a2158352f108bb4ff32350825081a9f29" +checksum = "2b350bfd03649e07aa05c0a81b3e15934374e585c98204a57e20b9d49f49bb9a" dependencies = [ "keyring-core", "log", @@ -186,12 +176,6 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" -[[package]] -name = "arrayref" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" - [[package]] name = "arrayvec" version = "0.7.8" @@ -210,7 +194,7 @@ dependencies = [ "nom", "num-traits", "rusticata-macros", - "thiserror 2.0.19", + "thiserror 2.0.20", "time", ] @@ -300,9 +284,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.43" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8" +checksum = "4f10dafd0c8d2e51ae9a748805777613ed0bbe17bf586b76c8311f45c020a32f" dependencies = [ "compression-codecs", "compression-core", @@ -430,15 +414,21 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.91" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.5", ] +[[package]] +name = "asyncband" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2d85fd3d291fabcc40c7232c92c280ec1754fd7b5d7ea769f143222143e179a" + [[package]] name = "atomic" version = "0.6.1" @@ -462,9 +452,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.10.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b180a3c8b55960db3426d8964b8745e652466a1a49fe1a2eda828046d30b5e4" +checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" dependencies = [ "aws-credential-types", "aws-runtime", @@ -505,9 +495,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.17.3" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -516,9 +506,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.43.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", @@ -529,9 +519,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9007227e10b5fed2f3e0a2beff489211e2b5604c400b7a9d5d81ca9d64c24bb" +checksum = "ef47857a1d4488b528f4a5d5715fa7c3300820897824152234d3fa22b1426657" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -554,9 +544,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.105.0" +version = "1.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ffd0fbe7873cb548a7aa60f9573c268fff94155397fd4f14dc9f1ecaaab8516" +checksum = "c3cfe74df5d9ad2fedd691973ad3521ebf4f27a3c68c792556686aedb5519bab" dependencies = [ "arc-swap", "aws-credential-types", @@ -580,9 +570,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.107.0" +version = "1.111.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175763eb222a46377df7aa257a3bca980ab3e96703fefc8f4d0b8da6ad2e254c" +checksum = "81b0ec31ed6191bd11350aae4b2004198f2db21350cb0a20c57e0a92e55dd161" dependencies = [ "arc-swap", "aws-credential-types", @@ -606,9 +596,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.110.0" +version = "1.114.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b14781dfbff48984017d57167b6ea0b6471c6920ec52b44a2677c7feb3c13" +checksum = "ef45745026107ec30c4ef86bd8ae4b002e7e5f6a86e4225240bdf6b06a0b944a" dependencies = [ "arc-swap", "aws-credential-types", @@ -644,11 +634,11 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac 0.13.0", + "hmac", "http 0.2.12", "http 1.5.0", "percent-encoding", - "sha2 0.11.0", + "sha2", "time", "tracing", ] @@ -687,9 +677,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.2.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e" +checksum = "ebfd138fac0337cee7516c352757ea73b9f2266e57d0bcb5bc70e9547e45aef1" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -744,9 +734,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.12.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07505b34e8f4b3591a4fa69e9792b52289b95488dbbc68c3c0075b7bedb245e1" +checksum = "b82e438d30e02a825d363bd639a9efaed68a8089d86101054b0081e7e0d3e606" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -770,9 +760,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.14.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b98f2e1fd67ec06618f9c291e5e495a468e60519e44c9c1979cd0521f3affdb" +checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", @@ -810,9 +800,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.6.1" +version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8" +checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9" dependencies = [ "base64-simd", "bytes", @@ -845,9 +835,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eec1cd5469f328c782dc3e33d4153cf118a54e33cbb3356d60d16f89883e1f94" +checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -964,7 +954,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.8.9", "object", "rustc-demangle", "windows-link 0.2.1", @@ -985,6 +975,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1036,25 +1032,30 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.13.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" dependencies = [ "serde_core", ] [[package]] name = "blake3" -version = "1.8.5" +version = "1.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" dependencies = [ - "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", ] [[package]] @@ -1077,18 +1078,18 @@ dependencies = [ [[package]] name = "block-padding" -version = "0.3.3" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +checksum = "710f1dd022ef4e93f8a438b4ba958de7f64308434fa6a87104481645cc30068b" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] name = "blocking" -version = "1.6.2" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +checksum = "a70e4329df6cb94385eed412ec92375c3cdd8a6e502493d1229b6414e4036dfa" dependencies = [ "async-channel", "async-task", @@ -1099,13 +1100,13 @@ dependencies = [ [[package]] name = "bollard" -version = "0.20.2" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4" +checksum = "dbe8358268799ebb3e4df23cb9d47f4c72bbc4f5247e2fa6a1bf7b6c0baea220" dependencies = [ "async-stream", - "base64", - "bitflags", + "base64 0.22.1", + "bitflags 2.13.2", "bollard-buildkit-proto", "bollard-stubs", "bytes", @@ -1123,7 +1124,7 @@ dependencies = [ "log", "num", "pin-project-lite", - "rand 0.9.5", + "rand 0.10.2", "rustls", "rustls-native-certs", "rustls-pki-types", @@ -1131,7 +1132,7 @@ dependencies = [ "serde_derive", "serde_json", "serde_urlencoded", - "thiserror 2.0.19", + "thiserror 2.0.20", "time", "tokio", "tokio-stream", @@ -1144,24 +1145,23 @@ dependencies = [ [[package]] name = "bollard-buildkit-proto" -version = "0.7.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a885520bf6249ab931a764ffdb87b0ceef48e6e7d807cfdb21b751e086e1ad" +checksum = "b5c97450e79c7c565302dd92e86b08823b47550fcb4fc5ce910194d1b087a1a3" dependencies = [ "prost", "prost-types", "tonic", "tonic-prost", - "ureq", ] [[package]] name = "bollard-stubs" -version = "1.52.1-rc.29.1.3" +version = "1.53.1-rc.29.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f0a8ca8799131c1837d1282c3f81f31e76ceb0ce426e04a7fe1ccee3287c066" +checksum = "ce412eb6f7096743011dc3cb5c674caeb24ced61d8c498fe07cf7998a4fea889" dependencies = [ - "base64", + "base64 0.22.1", "bollard-buildkit-proto", "bytes", "prost", @@ -1182,9 +1182,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" dependencies = [ "memchr", "regex-automata", @@ -1199,9 +1199,9 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytecheck" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0caa33a2c0edca0419d15ac723dff03f1956f7978329b1e3b5fdaaaed9d3ca8b" +checksum = "26333eeac754f0ad8a6bcd0eb0ac012156302e4e16b852b72ee399aea4f12c29" dependencies = [ "bytecheck_derive", "ptr_meta", @@ -1211,13 +1211,13 @@ dependencies = [ [[package]] name = "bytecheck_derive" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89385e82b5d1821d2219e0b095efa2cc1f246cbf99080f3be46a1a85c0d392d9" +checksum = "46d07918caa9eeaaf06b7873925c53a61daac173539b4f7715090745e44e4e69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -1259,9 +1259,9 @@ dependencies = [ [[package]] name = "bytesize" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "351a3e803ee3c6eaeee6b00076b767514b37c32a73d326c3ec7abddb7d6c3493" +checksum = "7354288c522e7e980fafd2075d63d1285794c3a6a16cdd492f189ea406e5f18b" [[package]] name = "castaway" @@ -1274,18 +1274,18 @@ dependencies = [ [[package]] name = "cbc" -version = "0.1.2" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +checksum = "ce2dc9ee5f88d11e0beb842c88b33c8a5cf0d1329c4b19494af42b07dbfe8896" dependencies = [ - "cipher 0.4.4", + "cipher", ] [[package]] name = "cc" -version = "1.4.0" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" dependencies = [ "find-msvc-tools", "jobserver", @@ -1307,12 +1307,12 @@ checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -1330,16 +1330,6 @@ dependencies = [ "windows-link 0.2.1", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common 0.1.7", - "inout 0.1.4", -] - [[package]] name = "cipher" version = "0.5.2" @@ -1348,14 +1338,14 @@ checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ "block-buffer 0.12.1", "crypto-common 0.2.2", - "inout 0.2.2", + "inout", ] [[package]] name = "clap" -version = "4.6.5" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", "clap_derive", @@ -1363,9 +1353,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.5" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstream", "anstyle", @@ -1382,7 +1372,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.5", ] [[package]] @@ -1406,12 +1396,6 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" -[[package]] -name = "cmsketch" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7ee2cfacbd29706479902b06d75ad8f1362900836aa32799eabc7e004bfd854" - [[package]] name = "color-print" version = "0.3.7" @@ -1445,14 +1429,14 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] name = "combine" -version = "4.6.7" +version = "4.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" dependencies = [ "bytes", "memchr", @@ -1475,22 +1459,22 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.38" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +checksum = "58a6d0db8759036a783bc7c3f7a07f8cef3bf9470eb1db3bc86e8bcd1c5d0fe8" dependencies = [ "compression-core", "flate2", "memchr", - "zstd", - "zstd-safe", + "zstd 0.14.0", + "zstd-safe 8.0.0", ] [[package]] name = "compression-core" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" +checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" [[package]] name = "concurrent-queue" @@ -1516,16 +1500,16 @@ dependencies = [ "serde-untagged", "serde_core", "serde_json", - "toml 1.1.4+spec-1.1.0", + "toml 1.1.6+spec-1.1.0", "winnow 1.0.4", "yaml-rust2", ] [[package]] name = "console" -version = "0.16.4" +version = "0.16.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" +checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3" dependencies = [ "encode_unicode", "libc", @@ -1601,6 +1585,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpubits" version = "0.1.1" @@ -1618,9 +1608,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] @@ -1637,18 +1627,18 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-epoch" -version = "0.9.20" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", ] @@ -1665,9 +1655,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crunchy" @@ -1702,7 +1692,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" dependencies = [ - "cipher 0.5.2", + "cipher", ] [[package]] @@ -1726,12 +1716,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" dependencies = [ - "darling_core 0.23.0", - "darling_macro 0.23.0", + "darling_core 0.24.1", + "darling_macro 0.24.1", ] [[package]] @@ -1749,15 +1739,15 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" dependencies = [ "ident_case", "proc-macro2", "quote", "strsim", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -1773,13 +1763,13 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ - "darling_core 0.23.0", + "darling_core 0.24.1", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -1798,9 +1788,46 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "datasketches" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.20", +] [[package]] name = "der-parser" @@ -1839,7 +1866,6 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "crypto-common 0.1.7", - "subtle", ] [[package]] @@ -1856,9 +1882,9 @@ dependencies = [ [[package]] name = "dirs" -version = "6.0.0" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +checksum = "8d57d423b3c82e89b9a24ca3091fee61f456a26edbd28d26c65906f4bc1dcd8f" dependencies = [ "dirs-sys", ] @@ -1872,7 +1898,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1883,7 +1909,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.5", ] [[package]] @@ -1901,7 +1927,7 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29547a1dc60885a552306986316bc9701ba120c1a8db6769fa68691529ad373d" dependencies = [ - "base64", + "base64 0.22.1", "serde", "serde_json", ] @@ -1926,7 +1952,7 @@ checksum = "f88959de2d447fd3eddcf1909d1f19fe084e27a056a6904203dc5d8b9e771c1e" dependencies = [ "rust_decimal", "serde", - "thiserror 2.0.19", + "thiserror 2.0.20", "time", "winnow 0.6.26", ] @@ -1939,9 +1965,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" [[package]] name = "encode_unicode" @@ -1951,11 +1977,17 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "encoding_rs" -version = "0.8.35" +version = "0.8.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +checksum = "7b5ef0006ac9ab233c38522f5ae99cae3625151de8f706cacee1cba4b8e2832a" dependencies = [ "cfg-if", + "core_detect", + "multiversion", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", ] [[package]] @@ -2030,7 +2062,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2065,10 +2097,11 @@ dependencies = [ [[package]] name = "eyre" -version = "0.6.12" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +checksum = "c08309dbcc659c5549a24ddb9b27027640641b282ef5768267c7e675558986a3" dependencies = [ + "autocfg", "indenter", "once_cell", ] @@ -2130,9 +2163,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" [[package]] name = "fixedbitset" @@ -2146,18 +2179,18 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags", + "bitflags 2.13.2", "rustc_version", ] [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.9.1", "zlib-rs", ] @@ -2199,18 +2232,18 @@ dependencies = [ [[package]] name = "foyer" -version = "0.22.3" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0abc0b87814989efa711f9becd9f26969820e2d3905db27d10969c4bd45890" +checksum = "6a59f42276891c0a4ce683fcda1aa1b4fd1065d68116c264e4bf49b9d318a0ed" dependencies = [ "anyhow", + "asyncband", "equivalent", "foyer-common", "foyer-memory", "foyer-storage", "foyer-tokio", "futures-util", - "mea", "mixtrics", "pin-project", "serde", @@ -2219,9 +2252,9 @@ dependencies = [ [[package]] name = "foyer-common" -version = "0.22.3" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3db80d5dece93adb7ad709c84578794724a9cba342a7e566c3551c7ec626789" +checksum = "643b47d510032a01e5af70dca288b0c5ec531646c1a8392e793fb5b0c13bef30" dependencies = [ "anyhow", "bincode", @@ -2246,21 +2279,21 @@ dependencies = [ [[package]] name = "foyer-memory" -version = "0.22.3" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db907f40a527ca2aa2f40a5f68b32ea58aa70f050cd233518e9ffd402cfba6ce" +checksum = "ae51f5089f3d9025ae77f17ea66d2489ac0f82fbb1d91462807bea174d486d82" dependencies = [ "anyhow", - "bitflags", - "cmsketch", + "asyncband", + "bitflags 2.13.2", + "datasketches", "equivalent", "foyer-common", "foyer-intrusive-collections", "foyer-tokio", "futures-util", - "hashbrown 0.16.1", - "itertools 0.14.0", - "mea", + "hashbrown 0.17.1", + "itertools 0.15.0", "mixtrics", "parking_lot", "paste", @@ -2271,12 +2304,13 @@ dependencies = [ [[package]] name = "foyer-storage" -version = "0.22.3" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1983f1db3d0710e9c9d5fc116d9202dccd41a2d1e032572224f1aff5520aa958" +checksum = "118192f532ba013f0cdc607efc22324b9ed855baed185608af0daa986e835c6b" dependencies = [ "allocator-api2", "anyhow", + "asyncband", "bytes", "core_affinity", "equivalent", @@ -2287,26 +2321,25 @@ dependencies = [ "fs4", "futures-core", "futures-util", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "io-uring", - "itertools 0.14.0", + "itertools 0.15.0", "libc", "lz4", - "mea", "parking_lot", "pin-project", - "rand 0.9.5", + "rand 0.10.2", "serde", "tracing", "twox-hash", - "zstd", + "zstd 0.13.3", ] [[package]] name = "foyer-tokio" -version = "0.22.3" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6577b05a7ffad0db555aedf00bfe52af818220fc4c1c3a7a12520896fc38627" +checksum = "6741d1133dfaab64d3f8a9290bbfed3685084add28f8b6ee7a6264aa4dc26918" dependencies = [ "tokio", ] @@ -2349,9 +2382,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -2364,9 +2397,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -2374,15 +2407,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -2391,9 +2424,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-lite" @@ -2410,26 +2443,26 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] name = "futures-sink" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-timer" @@ -2439,9 +2472,9 @@ checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -2510,6 +2543,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" dependencies = [ "polyval", + "zeroize", ] [[package]] @@ -2538,9 +2572,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.16" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -2548,7 +2582,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.5.0", - "indexmap 2.14.0", + "indexmap 2.14.2", "slab", "tokio", "tokio-util", @@ -2582,8 +2616,6 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ - "allocator-api2", - "equivalent", "foldhash 0.2.0", ] @@ -2621,9 +2653,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" [[package]] name = "hex" @@ -2633,20 +2665,11 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hkdf" -version = "0.12.4" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" dependencies = [ - "hmac 0.12.1", -] - -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest 0.10.7", + "hmac", ] [[package]] @@ -2711,9 +2734,9 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", @@ -2742,18 +2765,18 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" dependencies = [ "typenum", ] [[package]] name = "hyper" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", @@ -2820,7 +2843,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -2878,9 +2901,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -2892,22 +2915,43 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", + "serde", "tinystr", "writeable", "zerovec", ] [[package]] -name = "icu_normalizer" -version = "2.2.0" +name = "icu_locale_fallback" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -2919,16 +2963,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -2939,18 +2984,20 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", + "serde", + "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -2958,6 +3005,28 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_segmenter" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82d07aafccd67af15d02512a6adf5896fbc5ed00f2e99b471d2efa14016db3db" +dependencies = [ + "icu_collections", + "icu_locale_fallback", + "icu_provider", + "icu_segmenter_data", + "potential_utf", + "smallvec", + "utf8_iter", + "zerovec", +] + +[[package]] +name = "icu_segmenter_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae293c039020f9ec10710af98d29ce6aa2051486638b49c9a6409f3b4a9e98ad" + [[package]] name = "ident_case" version = "1.0.1" @@ -3004,9 +3073,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -3033,41 +3102,32 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" -[[package]] -name = "inout" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "block-padding", - "generic-array", -] - [[package]] name = "inout" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" dependencies = [ + "block-padding", "hybrid-array", ] [[package]] name = "io-uring" -version = "0.7.13" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0" +checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" dependencies = [ - "bitflags", + "bitflags 2.13.2", "cfg-if", "libc", ] [[package]] name = "ipnet" -version = "2.12.0" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "is_ci" @@ -3105,6 +3165,59 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link 0.2.1", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jni" version = "0.22.4" @@ -3117,7 +3230,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.19", + "thiserror 2.0.20", "walkdir", "windows-link 0.2.1", ] @@ -3166,9 +3279,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" dependencies = [ "cfg-if", "futures-util", @@ -3188,9 +3301,9 @@ dependencies = [ [[package]] name = "json_to_table" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f91246cf42b255a705436e360ca5894e376a50fdf696e4e60cb1bb6dd648e21" +checksum = "0ec64c9908ffa97b6ef6044d96c9f56de1c81643d3b9fbc2823e8847ab11b467" dependencies = [ "serde_json", "tabled", @@ -3198,9 +3311,9 @@ dependencies = [ [[package]] name = "keyring" -version = "4.1.6" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72585bb6cc9bc370d1d545b7e23fcce71dfd4461c5e15275e3cf51bdfd9a980a" +checksum = "2270074a3d26bcac93c1dc5d2845eb4c089e8d761ccf6e0ea266a16004640627" dependencies = [ "apple-native-keyring-store", "keyring-core", @@ -3231,9 +3344,9 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libredox" -version = "0.1.18" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" +checksum = "8d8f1ea3f21fd3405dcaf6c9b5c1630af9afc422d9073ea39c5f6d6c772e08ed" dependencies = [ "libc", ] @@ -3252,9 +3365,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "lock_api" @@ -3267,15 +3380,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lru" -version = "0.18.2" +version = "0.18.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" +checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" dependencies = [ "hashbrown 0.17.1", ] @@ -3339,15 +3452,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "mea" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31fc7d159de0085ab6dd7ff145a9819442cfd3d098f783263120503c3f3e58b0" -dependencies = [ - "slab", -] - [[package]] name = "memchr" version = "2.8.3" @@ -3410,6 +3514,15 @@ name = "miniz_oxide" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -3417,9 +3530,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", "wasi", @@ -3442,6 +3555,33 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "multiversion" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ca4bea16ffc3f443cf7d866912118196bfef4c6a1556ca00f9f9b00bb43f7c" +dependencies = [ + "multiversion-macros", +] + +[[package]] +name = "multiversion-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d416831a7317ef4b08bee00b69cbbb9c8763da7959a7026244d6266869f9c83" +dependencies = [ + "proc-macro2", + "quote", + "rustversion", + "syn 3.0.5", +] + +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "munge" version = "0.4.7" @@ -3468,7 +3608,7 @@ version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" dependencies = [ - "bitflags", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -3505,7 +3645,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3549,9 +3689,9 @@ checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -3602,7 +3742,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags", + "bitflags 2.13.2", ] [[package]] @@ -3632,7 +3772,7 @@ checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50" dependencies = [ "async-trait", "aws-lc-rs", - "base64", + "base64 0.22.1", "bytes", "chrono", "crc-fast", @@ -3656,7 +3796,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tracing", "url", @@ -3751,9 +3891,9 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "owo-colors" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" +checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8" [[package]] name = "papergrid" @@ -3797,9 +3937,9 @@ dependencies = [ [[package]] name = "parse-display" -version = "0.9.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +checksum = "e78deb158fb1d73b29efb4b7e9b9860b78059c670de06bd28df8d0b458ded0eb" dependencies = [ "parse-display-derive", "regex", @@ -3808,9 +3948,9 @@ dependencies = [ [[package]] name = "parse-display-derive" -version = "0.9.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +checksum = "8e95a50d1084dab562913062c4c34bb204b68fc6ec38a1395909ff5aaaf4f10a" dependencies = [ "proc-macro2", "quote", @@ -3857,11 +3997,11 @@ dependencies = [ [[package]] name = "pem" -version = "3.0.6" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" dependencies = [ - "base64", + "base64 0.23.1", "serde_core", ] @@ -3873,9 +4013,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.8" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df728be843c7070fab6ab7c328c4e9e9d78e23bf749c0669c86ee7ebfa050a2" +checksum = "6d45aeb61b4bf818e12d4205f2466f8c4748f85f4fce0146d1c03d69d753f0ad" dependencies = [ "memchr", "ucd-trie", @@ -3883,9 +4023,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.8" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e2dd6fc3b26b3462ee188aac870f5a41d398f1cd5e2408d16531bd71c9591fd" +checksum = "89cc5a242e25ed4e7704d0be240f2cfbe20a8c27e7e252d94835be93d92dc39f" dependencies = [ "pest", "pest_generator", @@ -3893,9 +4033,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.8" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a7a9205cfb6f596a9e8b689c0a15f9ceb7a1aafae7aaf788150ac65b29975b6" +checksum = "7abf21475cc3820fe4b2ca2dc2142902f67a02189f3b5b3a229f4febc01a43e5" dependencies = [ "pest", "pest_meta", @@ -3906,9 +4046,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.8.8" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85abd351c0de1e8384fc791a0737111a350394937e92b956b743dac12429f57c" +checksum = "adba4db388f687393c18c51348d44a41d870ca9df71a2c98172ea3035dc6936e" dependencies = [ "pest", ] @@ -3921,7 +4061,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap 2.14.0", + "indexmap 2.14.2", ] [[package]] @@ -3969,9 +4109,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "polling" @@ -3994,22 +4134,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" dependencies = [ "cpubits", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "universal-hash", + "zeroize", ] [[package]] name = "portable-atomic" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ + "serde_core", + "writeable", "zerovec", ] @@ -4074,29 +4226,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.13+spec-1.1.0", -] - -[[package]] -name = "proc-macro-error-attr2" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" -dependencies = [ - "proc-macro2", - "quote", -] - -[[package]] -name = "proc-macro-error2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" -dependencies = [ - "proc-macro-error-attr2", - "proc-macro2", - "quote", - "syn 2.0.119", + "toml_edit 0.25.15+spec-1.1.0", ] [[package]] @@ -4127,7 +4257,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" dependencies = [ - "bitflags", + "bitflags 2.13.2", "hex", "procfs-core", "rustix 0.38.44", @@ -4139,7 +4269,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" dependencies = [ - "bitflags", + "bitflags 2.13.2", "hex", ] @@ -4157,7 +4287,7 @@ dependencies = [ "parking_lot", "procfs", "protobuf", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -4168,7 +4298,7 @@ checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ "bit-set", "bit-vec 0.8.0", - "bitflags", + "bitflags 2.13.2", "num-traits", "rand 0.9.5", "rand_chacha", @@ -4252,22 +4382,22 @@ dependencies = [ [[package]] name = "ptr_meta" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" +checksum = "743da816b98c921cdbe8628ef7381b76f25ecf4da599fc80aca90eae7ef70cc0" dependencies = [ "ptr_meta_derive", ] [[package]] name = "ptr_meta_derive" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" +checksum = "1c8d9ca532f185d5d4db7a7c9d51420b452168ea1c2b913953281bd6fe1fcbd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -4300,7 +4430,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -4308,9 +4438,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.16" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" dependencies = [ "aws-lc-rs", "bytes", @@ -4323,7 +4453,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.19", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -4340,7 +4470,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4366,9 +4496,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rancor" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daff8b7b3ccf5f7ba270b3e7a0a4d4c701c5797e38dec27c7e2c3dbb830fed1c" +checksum = "9b534442d0fcdb55d66f373d9cac6d33b6293a2335bc2136dbd06ce0e87d2572" dependencies = [ "ptr_meta", ] @@ -4448,9 +4578,9 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.14.8" +version = "0.14.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055" +checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" dependencies = [ "aws-lc-rs", "pem", @@ -4466,7 +4596,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.13.2", ] [[package]] @@ -4477,27 +4607,27 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] name = "ref-cast" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.5", ] [[package]] @@ -4514,9 +4644,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -4552,11 +4682,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64", + "base64 0.23.1", "bytes", "futures-core", "futures-util", @@ -4607,14 +4737,14 @@ dependencies = [ [[package]] name = "rkyv" -version = "0.8.17" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "815cc8a37159a463064825246cadb07961e25cd9885908606f6d08a98d8f8874" +checksum = "d9776093b7ca170454ab1406954f7b7d97a57c51dc6c0642957fb2ef25c2d399" dependencies = [ "bytecheck", "bytes", "hashbrown 0.17.1", - "indexmap 2.14.0", + "indexmap 2.14.2", "munge", "ptr_meta", "rancor", @@ -4626,13 +4756,13 @@ dependencies = [ [[package]] name = "rkyv_derive" -version = "0.8.17" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ed1a78a1b19d184b0daa629dd9a024573173ec7d485b287cb369fb3607cc1c" +checksum = "1c25ef604ac7dd839d44d64648952ea23c97866f124ff671b0ed2cf3ad9bb06e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -4641,7 +4771,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81116b9531d61eabc41aeb228e4b6b2435bcca3233b98cf3b3077d4e6e9debb3" dependencies = [ - "bitflags", + "bitflags 2.13.2", "once_cell", "serde", "serde_derive", @@ -4662,9 +4792,9 @@ dependencies = [ [[package]] name = "rstest" -version = "0.26.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" +checksum = "948203e6d13b83e51a90d7cf236dd58a0065f23f4b902f00f306e7eb2bd09b9c" dependencies = [ "futures-timer", "futures-util", @@ -4673,9 +4803,9 @@ dependencies = [ [[package]] name = "rstest_macros" -version = "0.26.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" +checksum = "a8d92eaf7b6e51e12471d71ddc72608e7151573de44099aacfbb1128177c0da4" dependencies = [ "cfg-if", "glob", @@ -4691,12 +4821,12 @@ dependencies = [ [[package]] name = "rtoolbox" -version = "0.0.5" +version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50a0e551c1e27e1731aba276dbeaeac73f53c7cd34d1bda485d02bd1e0f36844" +checksum = "9a1efe12a1469752d0e6ff5ebec0b6ef4924cc5c4c71046b0ec730040535819d" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4711,9 +4841,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.42.1" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be2a24f50780bc85f09cc6ac299bdf1424302742d77221106859c9d8b102126a" +checksum = "7653272e75dcac41dc199fbea6f5797633994fafd339943c06c9af16bf29cd3a" dependencies = [ "arrayvec", "num-traits", @@ -4755,7 +4885,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys 0.4.15", @@ -4768,18 +4898,18 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.23.43" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "log", @@ -4831,7 +4961,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4842,9 +4972,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -4878,7 +5008,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s2-api" -version = "0.31.1" +version = "0.32.0" dependencies = [ "axum", "base64ct", @@ -4899,16 +5029,16 @@ dependencies = [ "serde", "serde_json", "strum", - "thiserror 2.0.19", + "thiserror 2.0.20", "time", "tokio-util", "utoipa", - "zstd", + "zstd 0.14.0", ] [[package]] name = "s2-cli" -version = "0.42.8" +version = "0.43.0" dependencies = [ "assert_cmd", "async-stream", @@ -4919,6 +5049,7 @@ dependencies = [ "clap", "color-print", "colored", + "compact_str", "config", "dirs", "fs2", @@ -4949,16 +5080,16 @@ dependencies = [ "serde", "serde_json", "serial_test", - "sha2 0.11.0", + "sha2", "strum", "tabled", "tempfile", "terminal_size", - "thiserror 2.0.19", + "thiserror 2.0.20", "tikv-jemallocator", "tokio", "tokio-stream", - "toml 1.1.4+spec-1.1.0", + "toml 1.1.6+spec-1.1.0", "tracing", "tracing-subscriber", "uuid", @@ -4968,7 +5099,7 @@ dependencies = [ [[package]] name = "s2-common" -version = "0.41.1" +version = "0.42.0" dependencies = [ "axum", "base64ct", @@ -4985,14 +5116,14 @@ dependencies = [ "serde", "serde_json", "strum", - "thiserror 2.0.19", + "thiserror 2.0.20", "time", "utoipa", ] [[package]] name = "s2-lite" -version = "0.42.8" +version = "0.43.0" dependencies = [ "async-stream", "async-trait", @@ -5007,7 +5138,7 @@ dependencies = [ "eyre", "futures", "http 1.5.0", - "indexmap 2.14.0", + "indexmap 2.14.2", "itertools 0.15.0", "parking_lot", "prometheus", @@ -5025,13 +5156,13 @@ dependencies = [ "serde_json", "slatedb", "strum", - "thiserror 2.0.19", + "thiserror 2.0.20", "tikv-jemallocator", "time", "tokio", "tokio-util", "tower", - "tower-http 0.7.0", + "tower-http 0.7.1", "tracing", "tracing-subscriber", "utoipa", @@ -5040,8 +5171,9 @@ dependencies = [ [[package]] name = "s2-resource-spec" -version = "0.2.2" +version = "0.3.0" dependencies = [ + "compact_str", "humantime", "s2-common", "schemars 1.2.2", @@ -5051,13 +5183,14 @@ dependencies = [ [[package]] name = "s2-sdk" -version = "0.34.4" +version = "0.35.0" dependencies = [ "assert_matches", "async-compression", "async-stream", "async-trait", "bytes", + "compact_str", "futures-core", "futures-util", "h2", @@ -5080,7 +5213,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "test-context", - "thiserror 2.0.19", + "thiserror 2.0.20", "time", "tokio", "tokio-muxt", @@ -5094,7 +5227,7 @@ dependencies = [ [[package]] name = "s2-storage" -version = "0.2.4" +version = "0.2.6" dependencies = [ "aegis", "aes-gcm", @@ -5106,17 +5239,17 @@ dependencies = [ "s2-common", "secrecy", "serde", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] name = "s2-testcontainers" -version = "0.42.8" +version = "0.43.0" dependencies = [ "reqwest", "s2-sdk", "testcontainers", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", ] @@ -5172,7 +5305,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 3.0.3", + "syn 3.0.5", ] [[package]] @@ -5192,20 +5325,20 @@ dependencies = [ [[package]] name = "secret-service" -version = "5.1.0" +version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a62d7f86047af0077255a29494136b9aaaf697c76ff70b8e49cded4e2623c14" +checksum = "5107b24b91445dd2aa449a258a1807b63240942157292354dc5bfdbeb8bc6db8" dependencies = [ - "aes 0.8.4", + "aes", "cbc", "futures-util", - "generic-array", - "getrandom 0.2.17", + "getrandom 0.4.3", "hkdf", + "hybrid-array", "num", "once_cell", "serde", - "sha2 0.10.9", + "sha2", "zbus", ] @@ -5215,7 +5348,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.13.2", "core-foundation", "core-foundation-sys", "libc", @@ -5288,7 +5421,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.5", ] [[package]] @@ -5299,7 +5432,7 @@ checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.5", ] [[package]] @@ -5308,7 +5441,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "itoa", "memchr", "serde", @@ -5335,7 +5468,7 @@ checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.5", ] [[package]] @@ -5370,16 +5503,17 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.21.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4" dependencies = [ - "base64", + "base64 0.23.1", "bs58", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.14.0", + "indexmap 2.14.2", + "jiff", "schemars 0.9.0", "schemars 1.2.2", "serde_core", @@ -5390,14 +5524,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.21.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc" dependencies = [ - "darling 0.23.0", + "darling 0.24.1", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -5406,7 +5540,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "itoa", "ryu", "serde", @@ -5435,7 +5569,7 @@ checksum = "a22144e767da4ddd8416dbf383700542ffd8a5dc493dfecedfe1fe3ad03c98ae" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.5", ] [[package]] @@ -5449,17 +5583,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", -] - [[package]] name = "sha2" version = "0.11.0" @@ -5467,7 +5590,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "digest 0.11.3", ] @@ -5532,15 +5655,15 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "slatedb" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ca56b01922b15aa69fe3abb62cadc985d86032c9647e4606e211c4da751a76" +checksum = "fb40332f41e231926df3a0ef742c05316b43368ba4b520433d2a1eba1ab00f0f" dependencies = [ "async-channel", "async-trait", "atomic", "backon", - "bitflags", + "bitflags 2.13.2", "bytes", "chrono", "crc32fast", @@ -5574,14 +5697,14 @@ dependencies = [ "url", "uuid", "walkdir", - "zstd", + "zstd 0.13.3", ] [[package]] name = "slatedb-common" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa8de522ff46a0f9b5a66f45e650d125421d4d91990933591092f9d010c40d1" +checksum = "6b77f238f348e95e1653a5235f880ac703011b82582bd556f2aba405abf8180e" dependencies = [ "chrono", "log", @@ -5595,9 +5718,9 @@ dependencies = [ [[package]] name = "slatedb-txn-obj" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85fd9c0c86dd4954524fa8238d0548bd80f4a9a66983cbde3728402d339993e" +checksum = "1ad63b6c250219e26d71f497820f970274c301f896366daa56915f2a05df0e0a" dependencies = [ "async-trait", "bytes", @@ -5618,9 +5741,9 @@ checksum = "88414a5ca1f85d82cc34471e975f0f74f6aa54c40f062efa42c0080e7f763f81" [[package]] name = "smallvec" -version = "1.15.2" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" [[package]] name = "socket2" @@ -5746,9 +5869,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.3" +version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" dependencies = [ "proc-macro2", "quote", @@ -5791,9 +5914,9 @@ dependencies = [ [[package]] name = "tabled" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5dc662e6da844ad6e428ad16b57967c9d33c82e16bb1c258326c0c078605dff" +checksum = "2d2596a104db1900b943f97d793a6c99addca918c4525c999f751a0f17d472eb" dependencies = [ "papergrid", "tabled_derive", @@ -5802,12 +5925,11 @@ dependencies = [ [[package]] name = "tabled_derive" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea5d1b13ca6cff1f9231ffd62f15eefd72543dab5e468735f1a456728a02846" +checksum = "7dca1937322a1e892b1a65f6a6736183bb0a29d3b4234d1d53bc436dff9beb76" dependencies = [ "heck 0.5.0", - "proc-macro-error2", "proc-macro2", "quote", "syn 2.0.119", @@ -5820,10 +5942,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5833,7 +5955,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5865,9 +5987,9 @@ dependencies = [ [[package]] name = "testcontainers" -version = "0.27.3" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfd5785b5483672915ed5fe3cddf9f546802779fc1eceff0a6fb7321fac81c1e" +checksum = "6e2bbe381afaaa58ea610c5fc3ffb2184063a32b3e358a179f0b4865dd59934a" dependencies = [ "astral-tokio-tar", "async-trait", @@ -5887,7 +6009,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tokio-stream", "tokio-util", @@ -5905,11 +6027,11 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.16.2" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +checksum = "b81c0cb5fce14f53e49c1d4da0c508334ff12040221bb8ab01b2dabd91d04b6e" dependencies = [ - "unicode-linebreak", + "icu_segmenter", "unicode-width 0.2.2", ] @@ -5924,11 +6046,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.19", + "thiserror-impl 2.0.20", ] [[package]] @@ -5944,13 +6066,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.5", ] [[package]] @@ -5984,9 +6106,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.54" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", "num-conv", @@ -6023,19 +6145,20 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] [[package]] name = "tinyvec" -version = "1.12.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" dependencies = [ "tinyvec_macros", ] @@ -6071,7 +6194,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.5", ] [[package]] @@ -6086,9 +6209,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.4" +version = "0.26.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" dependencies = [ "rustls", "tokio", @@ -6157,11 +6280,11 @@ dependencies = [ [[package]] name = "toml" -version = "1.1.4+spec-1.1.0" +version = "1.1.6+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" +checksum = "920602543f0911ab71da12c50d59701da54c196d1a2bf5cb4b75667f137a406a" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "serde_core", "serde_spanned 1.1.1", "toml_datetime 1.1.1+spec-1.1.0", @@ -6194,7 +6317,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "serde", "serde_spanned 0.6.9", "toml_datetime 0.6.11", @@ -6204,11 +6327,11 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.13+spec-1.1.0" +version = "0.25.15+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "winnow 1.0.4", @@ -6243,7 +6366,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", - "base64", + "base64 0.22.1", "bytes", "h2", "http 1.5.0", @@ -6283,7 +6406,7 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "indexmap 2.14.0", + "indexmap 2.14.2", "pin-project-lite", "slab", "sync_wrapper", @@ -6300,7 +6423,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags", + "bitflags 2.13.2", "bytes", "futures-util", "http 1.5.0", @@ -6314,12 +6437,12 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" +checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c" dependencies = [ "async-compression", - "bitflags", + "bitflags 2.13.2", "bytes", "futures-core", "http 1.5.0", @@ -6416,12 +6539,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twox-hash" -version = "2.1.3" +version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" -dependencies = [ - "rand 0.10.2", -] +checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" [[package]] name = "typed-path" @@ -6490,12 +6610,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-linebreak" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" - [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -6548,33 +6662,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "ureq" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" -dependencies = [ - "base64", - "log", - "percent-encoding", - "rustls", - "rustls-pki-types", - "ureq-proto", - "utf8-zero", -] - -[[package]] -name = "ureq-proto" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" -dependencies = [ - "base64", - "http 1.5.0", - "httparse", - "log", -] - [[package]] name = "url" version = "2.5.8" @@ -6594,12 +6681,6 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" -[[package]] -name = "utf8-zero" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -6615,9 +6696,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "utoipa" version = "5.4.0" -source = "git+https://github.com/infiniteregrets/utoipa?rev=9cd181d40ac0a50551ebe1995a4d73e8685890d6#9cd181d40ac0a50551ebe1995a4d73e8685890d6" +source = "git+https://github.com/infiniteregrets/utoipa?rev=82bcb28a792ba9a0d29963827ec473823099fc94#82bcb28a792ba9a0d29963827ec473823099fc94" dependencies = [ - "indexmap 2.14.0", + "indexmap 2.14.2", "serde", "serde_json", "utoipa-gen", @@ -6626,7 +6707,7 @@ dependencies = [ [[package]] name = "utoipa-gen" version = "5.4.0" -source = "git+https://github.com/infiniteregrets/utoipa?rev=9cd181d40ac0a50551ebe1995a4d73e8685890d6#9cd181d40ac0a50551ebe1995a4d73e8685890d6" +source = "git+https://github.com/infiniteregrets/utoipa?rev=82bcb28a792ba9a0d29963827ec473823099fc94#82bcb28a792ba9a0d29963827ec473823099fc94" dependencies = [ "proc-macro2", "quote", @@ -6635,9 +6716,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.24.0" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ "getrandom 0.4.3", "js-sys", @@ -6709,9 +6790,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" dependencies = [ "cfg-if", "once_cell", @@ -6722,9 +6803,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.76" +version = "0.4.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" dependencies = [ "js-sys", "wasm-bindgen", @@ -6732,9 +6813,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6742,22 +6823,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" dependencies = [ "unicode-ident", ] @@ -6777,9 +6858,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.103" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" dependencies = [ "js-sys", "wasm-bindgen", @@ -6826,7 +6907,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -7122,9 +7203,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "x509-parser" @@ -7140,7 +7221,7 @@ dependencies = [ "nom", "oid-registry", "rusticata-macros", - "thiserror 2.0.19", + "thiserror 2.0.20", "time", ] @@ -7168,9 +7249,9 @@ checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "yaml-rust2" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "631a50d867fafb7093e709d75aaee9e0e0d5deb934021fcea25ac2fe09edc51e" +checksum = "b36710ce3a279cfce8465dbab826f161675a262950b922cb2c3663852dfe9eb0" dependencies = [ "arraydeque", "encoding_rs", @@ -7218,9 +7299,9 @@ dependencies = [ [[package]] name = "zbus" -version = "5.18.0" +version = "5.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe18fb60dc696039e738717b76eaea21e7a4489bbb1885020b43c94236d7e98a" +checksum = "5db4be7c075cb421e4b7ee645541604239bd243ba7c357511f4ff3a74b555907" dependencies = [ "async-broadcast", "async-executor", @@ -7253,9 +7334,9 @@ dependencies = [ [[package]] name = "zbus-secret-service-keyring-store" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ccede190ba363386a24e8021c7f3848393976609ec9f5d1f8c6c09ef37075b4" +checksum = "74801d001b9e7729adb4f1825b67b398185fed424749aa3d8bacf70417137d9a" dependencies = [ "keyring-core", "secret-service", @@ -7264,14 +7345,14 @@ dependencies = [ [[package]] name = "zbus_macros" -version = "5.18.0" +version = "5.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe96480bed92df2b442a1a30df364e12d08eed03aeb061f2b8dc6afb2be91119" +checksum = "2990635d09ade6df1868f72f8cac69a876a90981e8bd3c40b1be413f8dc88f40" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", "zbus_names", "zvariant", "zvariant_utils", @@ -7289,19 +7370,28 @@ dependencies = [ ] [[package]] -name = "zerocopy" -version = "0.8.55" +name = "zcheapstr" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +checksum = "d1afec51604565183aeb5c54c20aeab286120d4e4460f7f76e3e8bb8c0d99473" +dependencies = [ + "serde", +] + +[[package]] +name = "zerocopy" +version = "0.8.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.55" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" dependencies = [ "proc-macro2", "quote", @@ -7337,21 +7427,23 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", "zerofrom", + "zerovec", ] [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", @@ -7359,13 +7451,13 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", ] [[package]] @@ -7376,7 +7468,7 @@ checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" dependencies = [ "crc32fast", "flate2", - "indexmap 2.14.0", + "indexmap 2.14.2", "memchr", "typed-path", "zopfli", @@ -7384,9 +7476,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" [[package]] name = "zmij" @@ -7412,23 +7504,41 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe", + "zstd-safe 7.3.0", +] + +[[package]] +name = "zstd" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e" +dependencies = [ + "zstd-safe 8.0.0", ] [[package]] name = "zstd-safe" -version = "7.2.4" +version = "7.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-safe" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.0+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" dependencies = [ "cc", "pkg-config", @@ -7436,40 +7546,41 @@ dependencies = [ [[package]] name = "zvariant" -version = "5.13.1" +version = "5.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee2a0bcd2a907786a456fff45aaaaf54c9ba5f50b71ae9ec1a4edd200c94911" +checksum = "c1d34c27cc6cdd1f458427519dd6b8612f7b7e3f7b9a0b2355d041dda9869147" dependencies = [ "endi", "enumflags2", "serde", "winnow 1.0.4", + "zcheapstr", "zvariant_derive", "zvariant_utils", ] [[package]] name = "zvariant_derive" -version = "5.13.1" +version = "5.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38a708216a18780796770bfe3f4739c7c83a3e8f789b755534bbbc06e4e23e12" +checksum = "864155e69b4352db0c7f374917bf45d1e0c8d17659c8b3dbf9795f3673f8c497" dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", "zvariant_utils", ] [[package]] name = "zvariant_utils" -version = "3.5.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90cb9383f9b45290407a1258b202d3f8f01db719eb60b4e4055c6375af4fc7c7" +checksum = "bad0294361a320b694a328460dc73add56c306150f5cb6bfafc44446120008a3" dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.119", + "syn 3.0.5", "winnow 1.0.4", ] diff --git a/Cargo.toml b/Cargo.toml index a092a9b7..a76dfc09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,14 +29,14 @@ axum-server = "0.8" base64ct = "1.8" blake3 = "1.8" bytes = "1" -bytesize = "2.6" +bytesize = "2.7" clap = "4.6" color-print = "0.3" colored = "3.1" compact_str = "0.10" config = "0.15" dashmap = "6.2" -dirs = "6.0" +dirs = "7.0" enumset = "1.1" eyre = "0.6" flate2 = "1.1" @@ -49,8 +49,8 @@ humantime = "2.4" indexmap = "2.14" indicatif = "0.18" itertools = "0.15" -json_to_table = "0.13" -keyring = "4.1" +json_to_table = "0.14" +keyring = "4.2" miette = "7.6" mime = "0.3" parking_lot = "0.12" @@ -63,24 +63,24 @@ rcgen = { version = "0.14", default-features = false, features = ["crypto", "pem reqwest = { version = "0.13", default-features = false, features = ["rustls"] } rkyv = "0.8" rpassword = "7.5" -rstest = "0.26" +rstest = "0.27" rustls = { version = "0.23", default-features = false, features = ["logging", "std", "tls12"] } -s2-api = { path = "api", version = "0.31" } -s2-common = { path = "common", version = "0.41" } -s2-lite = { path = "lite", version = "0.42" } -s2-resource-spec = { path = "resource-spec", version = "0.2" } -s2-sdk = { path = "sdk", version = "0.34" } +s2-api = { path = "api", version = "0.32" } +s2-common = { path = "common", version = "0.42" } +s2-lite = { path = "lite", version = "0.43" } +s2-resource-spec = { path = "resource-spec", version = "0.3" } +s2-sdk = { path = "sdk", version = "0.35" } s2-storage = { path = "storage", version = "0.2" } schemars = "1.2" secrecy = "0.10.3" semver = "1.0" serde = "1.0" serde_json = "1.0" -slatedb = "0.15.0" +slatedb = "0.16.0" strum = "0.28" -tabled = "0.21" +tabled = "0.22" tempfile = "3.27" -testcontainers = "0.27" +testcontainers = "0.28" thiserror = "2.0" tikv-jemallocator = { version = "0.7", features = ["unprefixed_malloc_on_supported_platforms"] } time = "0.3" @@ -92,12 +92,12 @@ tower-http = "0.7" tracing = "0.1" tracing-subscriber = "0.3" utoipa = "=5.4" -uuid = "1.24" +uuid = "1.26" xxhash-rust = "0.8" -zstd = "0.13" +zstd = "0.14" [patch.crates-io] -utoipa = { git = "https://github.com/infiniteregrets/utoipa", rev = "9cd181d40ac0a50551ebe1995a4d73e8685890d6" } +utoipa = { git = "https://github.com/infiniteregrets/utoipa", rev = "82bcb28a792ba9a0d29963827ec473823099fc94" } [profile.dev] panic = "abort" diff --git a/README.md b/README.md index 9c3eb14e..6e3acc97 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,34 @@ nc starwars.s2.dev 23 | s2 append s2://liteness/starwars Deploy `s2-lite` to Kubernetes using Helm. See the [Helm chart documentation](charts/s2-lite-helm/README.md) for installation instructions and configuration options. +### Storage + +Lite persists to an S3-compatible bucket or a local directory, or runs in-memory when neither is given. +The write-ahead log (WAL) shares the main store by default; it can be placed in a separate bucket or +directory to isolate WAL latency from flushes and compaction. `--path` applies to both stores. + +| Setting | Main store | WAL store | +| --- | --- | --- | +| S3 bucket | `--bucket` | `--wal-bucket` / `S2LITE_WAL_BUCKET` | +| Local directory | `--local-root` | `--wal-local-root` / `S2LITE_WAL_LOCAL_ROOT` | +| S3 endpoint | `AWS_ENDPOINT_URL_S3` | `S2LITE_WAL_AWS_ENDPOINT_URL_S3` | +| AWS region | `AWS_REGION` | `S2LITE_WAL_AWS_REGION` | +| Static credentials | `AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY` | `S2LITE_WAL_AWS_ACCESS_KEY_ID` + `S2LITE_WAL_AWS_SECRET_ACCESS_KEY` | +| Session token | `AWS_SESSION_TOKEN` | `S2LITE_WAL_AWS_SESSION_TOKEN` | + +Without static credentials, the standard AWS credential chain (profile, instance role, etc.) is used. +The WAL bucket inherits the main store's S3 settings; each `S2LITE_WAL_AWS_*` variable overrides just +that setting, except that a WAL key pair replaces the main credentials (and session token) as a set. +A WAL store requires a persistent main store. + +```bash +# LSM in S3, WAL on local disk +s2 lite --bucket my-bucket --wal-local-root /data/wal + +# LSM and WAL in separate buckets +s2 lite --bucket my-bucket --wal-bucket my-wal-bucket +``` + ### Monitoring `/health` will return 200 on success for readiness and liveness checks @@ -209,7 +237,7 @@ Deploy `s2-lite` to Kubernetes using Helm. See the [Helm chart documentation](ch Use `SL8_` prefixed environment variables, e.g.: ```bash -# Defaults to 50ms for remote bucket / 5ms in-memory +# Defaults to 50ms for S3, 5ms otherwise; follows the WAL store when set SL8_FLUSH_INTERVAL=10ms ``` diff --git a/api/CHANGELOG.md b/api/CHANGELOG.md index 4c989e96..55052994 100644 --- a/api/CHANGELOG.md +++ b/api/CHANGELOG.md @@ -2,6 +2,51 @@ All notable changes to this project will be documented in this file. +## [0.32.0] - 2026-09-25 + +### Features + +- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) + +### Bug Fixes + +- Skip s2s response compression the client refused with q=0 ([#781](https://github.com/s2-streamstore/s2/issues/781)) + + + +## [0.31.5] - 2026-09-22 + +### Bug Fixes + +- Preserve SSE read budgets across reconnects ([#727](https://github.com/s2-streamstore/s2/issues/727)) + + + +## [0.31.4] - 2026-09-16 + +### Miscellaneous Tasks + +- Sync specs submodule ([#739](https://github.com/s2-streamstore/s2/issues/739)) + + + +## [0.31.3] - 2026-09-11 + +### Features + +- `s2-stream-config` header for auto-created streams ([#718](https://github.com/s2-streamstore/s2/issues/718)) + + + +## [0.31.2] - 2026-09-10 + +### Bug Fixes + +- Reject NUL bytes in stream names and access token IDs ([#728](https://github.com/s2-streamstore/s2/issues/728)) +- Switch JSON extractor from sonic-rs back to serde_json ([#729](https://github.com/s2-streamstore/s2/issues/729)) + + + ## [0.31.1] - 2026-08-13 ### Features diff --git a/api/Cargo.toml b/api/Cargo.toml index 691ccc7a..35cd9cee 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-api" -version = "0.31.1" +version = "0.32.0" description = "API types for S2, the durable streams API" edition.workspace = true license.workspace = true diff --git a/api/specs b/api/specs index 973e0d92..edaa1fbc 160000 --- a/api/specs +++ b/api/specs @@ -1 +1 @@ -Subproject commit 973e0d92166ee1386b71d4e7ade8bd0fc2aaf4b8 +Subproject commit edaa1fbcb2507362d6c680c66cdc88660e73e036 diff --git a/api/src/data.rs b/api/src/data.rs index f9fc4fc7..9e53a92f 100644 --- a/api/src/data.rs +++ b/api/src/data.rs @@ -96,6 +96,25 @@ pub struct S2FormatHeader { pub s2_format: Format, } +#[rustfmt::skip] +#[derive(Debug)] +#[cfg_attr(feature = "utoipa", derive(utoipa::IntoParams))] +#[cfg_attr(feature = "utoipa", into_params(parameter_in = Header))] +pub struct S2StreamConfigHeader { + /// JSON-encoded `StreamConfig` to apply if the stream is created on append or read. + /// Unset fields inherit the basin's default stream configuration. + /// Ignored if the stream already exists. + /// Compact JSON is preferred. + #[cfg_attr(feature = "utoipa", param( + required = false, + rename = "s2-stream-config", + content_type = "application/json", + value_type = crate::v1::config::StreamConfig, + example = json!({"retention_policy":{"age":3600},"delete_on_empty":{"min_age_secs":300}}), + ))] + pub s2_stream_config: String, +} + #[rustfmt::skip] #[derive(Debug)] #[cfg_attr(feature = "utoipa", derive(utoipa::IntoParams))] @@ -447,7 +466,7 @@ pub mod extract { // StreamReconfiguration: exercises Maybe in all three states use s2_common::maybe::Maybe; - use crate::v1::config::{StorageClass, TimestampingMode, TimestampingReconfiguration}; + use crate::v1::config::{TimestampingMode, TimestampingReconfiguration}; // All fields unspecified (empty JSON object) assert_roundtrip(&StreamReconfiguration { @@ -458,7 +477,7 @@ pub mod extract { }); // Mix of specified-null and specified-value assert_roundtrip(&StreamReconfiguration { - storage_class: Maybe::Specified(Some(StorageClass::Express)), + storage_class: Maybe::Specified(Some("express".into())), retention_policy: Maybe::Specified(None), timestamping: Maybe::Specified(Some(TimestampingReconfiguration { mode: Maybe::Specified(Some(TimestampingMode::ClientRequire)), diff --git a/api/src/v1/config.rs b/api/src/v1/config.rs index fc523439..671c0cb8 100644 --- a/api/src/v1/config.rs +++ b/api/src/v1/config.rs @@ -1,37 +1,10 @@ -use std::time::Duration; +use std::{str::FromStr, time::Duration}; -use s2_common::maybe::Maybe; +use compact_str::CompactString; +use http::{HeaderName, HeaderValue}; +use s2_common::{http::ParseableHeader, maybe::Maybe}; use serde::{Deserialize, Serialize}; -#[rustfmt::skip] -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] -#[serde(rename_all = "kebab-case")] -pub enum StorageClass { - /// Append tail latency under 400 milliseconds with s2.dev. - Standard, - /// Append tail latency under 40 milliseconds with s2.dev. - Express, -} - -impl From for s2_common::config::StorageClass { - fn from(value: StorageClass) -> Self { - match value { - StorageClass::Express => Self::Express, - StorageClass::Standard => Self::Standard, - } - } -} - -impl From for StorageClass { - fn from(value: s2_common::config::StorageClass) -> Self { - match value { - s2_common::config::StorageClass::Express => Self::Express, - s2_common::config::StorageClass::Standard => Self::Standard, - } - } -} - #[rustfmt::skip] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] @@ -298,8 +271,9 @@ impl From for EncryptionAlgorithm { #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] pub struct StreamConfig { - /// Storage class for recent writes. - pub storage_class: Option, + /// [Storage class](https://s2.dev/docs/storage-classes) for recent writes. + #[cfg_attr(feature = "utoipa", schema(value_type = Option))] + pub storage_class: Option, /// Retention policy for the stream. /// If unspecified, the default is to retain records for 7 days. pub retention_policy: Option, @@ -320,7 +294,7 @@ impl StreamConfig { } = config; let config = StreamConfig { - storage_class: storage_class.map(Into::into), + storage_class, retention_policy: retention_policy.map(Into::into), timestamping: TimestampingConfig::to_opt(timestamping), delete_on_empty: DeleteOnEmptyConfig::to_opt(delete_on_empty), @@ -331,6 +305,12 @@ impl StreamConfig { Some(config) } } + + /// Encode as compact JSON for the `s2-stream-config` header. + pub fn to_header_value(&self) -> HeaderValue { + let json = serde_json::to_string(self).expect("StreamConfig serializes to JSON"); + HeaderValue::from_str(&json).expect("compact JSON of StreamConfig is a valid header value") + } } impl From for StreamConfig { @@ -343,7 +323,7 @@ impl From for StreamConfig { } = value; Self { - storage_class: Some(storage_class.into()), + storage_class, retention_policy: Some(retention_policy.into()), timestamping: Some(timestamping.into()), delete_on_empty: Some(delete_on_empty.into()), @@ -351,6 +331,30 @@ impl From for StreamConfig { } } +pub static STREAM_CONFIG_HEADER: HeaderName = HeaderName::from_static("s2-stream-config"); + +/// Value of the `s2-stream-config` header: a JSON-encoded [`StreamConfig`] to apply over the +/// basin's default stream config if the stream is created on append or read. Ignored if the +/// stream already exists. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StreamConfigHeader(pub s2_common::config::OptionalStreamConfig); + +impl FromStr for StreamConfigHeader { + type Err = s2_common::ValidationError; + + fn from_str(s: &str) -> Result { + let config: StreamConfig = + serde_json::from_str(s).map_err(|e| format!("invalid JSON: {e}"))?; + Ok(Self(config.try_into()?)) + } +} + +impl ParseableHeader for StreamConfigHeader { + fn name() -> &'static HeaderName { + &STREAM_CONFIG_HEADER + } +} + impl TryFrom for s2_common::config::OptionalStreamConfig { type Error = s2_common::ValidationError; @@ -368,7 +372,7 @@ impl TryFrom for s2_common::config::OptionalStreamConfig { }; let config = Self { - storage_class: storage_class.map(Into::into), + storage_class, retention_policy, timestamping: timestamping.map(Into::into).unwrap_or_default(), delete_on_empty: delete_on_empty.map(Into::into).unwrap_or_default(), @@ -382,10 +386,10 @@ impl TryFrom for s2_common::config::OptionalStreamConfig { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] pub struct StreamReconfiguration { - /// Storage class for recent writes. + /// [Storage class](https://s2.dev/docs/storage-classes) for recent writes. #[serde(default, skip_serializing_if = "Maybe::is_unspecified")] - #[cfg_attr(feature = "utoipa", schema(value_type = Option))] - pub storage_class: Maybe>, + #[cfg_attr(feature = "utoipa", schema(value_type = Option))] + pub storage_class: Maybe>, /// Retention policy for the stream. /// If unspecified, the default is to retain records for 7 days. #[serde(default, skip_serializing_if = "Maybe::is_unspecified")] @@ -413,7 +417,7 @@ impl TryFrom for s2_common::config::StreamReconfiguration } = value; Ok(Self { - storage_class: storage_class.map_opt(Into::into), + storage_class, retention_policy: retention_policy.try_map_opt(TryInto::try_into)?, timestamping: timestamping.map_opt(Into::into), delete_on_empty: delete_on_empty.map_opt(Into::into), @@ -431,7 +435,7 @@ impl From for StreamReconfiguration { } = value; Self { - storage_class: storage_class.map_opt(Into::into), + storage_class, retention_policy: retention_policy.map_opt(Into::into), timestamping: timestamping.map_opt(Into::into), delete_on_empty: delete_on_empty.map_opt(Into::into), @@ -566,8 +570,8 @@ mod tests { use super::*; - fn gen_storage_class() -> impl Strategy { - prop_oneof![Just(StorageClass::Standard), Just(StorageClass::Express)] + fn gen_storage_class() -> impl Strategy { + "[a-z][a-z0-9-]{0,30}".prop_map(CompactString::from) } fn gen_timestamping_mode() -> impl Strategy { @@ -724,7 +728,7 @@ mod tests { ) .prop_map(|(sc, rp, ts_mode, ts_uncapped, doe)| { s2_common::config::OptionalStreamConfig { - storage_class: sc.map(Into::into), + storage_class: sc, retention_policy: rp.map(|rp| match rp { RetentionPolicy::Age(secs) => { s2_common::config::RetentionPolicy::Age(Duration::from_secs(secs)) @@ -796,7 +800,7 @@ mod tests { prop_assert_eq!( merged.storage_class, - stream.storage_class.or(basin.storage_class).unwrap_or_default() + stream.storage_class.or(basin.storage_class) ); prop_assert_eq!( merged.retention_policy, @@ -852,7 +856,7 @@ mod tests { new_rp_secs in 1u64..u64::MAX, ) { let reconfig = s2_common::config::StreamReconfiguration { - storage_class: Maybe::Specified(Some(new_sc.into())), + storage_class: Maybe::Specified(Some(new_sc.clone())), retention_policy: Maybe::Specified(Some( s2_common::config::RetentionPolicy::Age(Duration::from_secs(new_rp_secs)) )), @@ -860,7 +864,7 @@ mod tests { }; let result = base.reconfigure(reconfig); - prop_assert_eq!(result.storage_class, Some(new_sc.into())); + prop_assert_eq!(result.storage_class, Some(new_sc)); prop_assert_eq!( result.retention_policy, Some(s2_common::config::RetentionPolicy::Age(Duration::from_secs(new_rp_secs))) @@ -875,7 +879,7 @@ mod tests { ) { // non-default storage class -> Some let internal = s2_common::config::OptionalStreamConfig { - storage_class: Some(sc.into()), + storage_class: Some(sc), ..Default::default() }; prop_assert!(StreamConfig::to_opt(internal).is_some()); @@ -923,7 +927,7 @@ mod tests { ) { let base = s2_common::config::BasinConfig { default_stream_config: s2_common::config::OptionalStreamConfig { - storage_class: base_sc.map(Into::into), + storage_class: base_sc, ..Default::default() }, stream_cipher: base_algorithm.map(Into::into), @@ -954,7 +958,7 @@ mod tests { let reconfig = s2_common::config::BasinReconfiguration { default_stream_config: Maybe::Specified(Some(s2_common::config::StreamReconfiguration { - storage_class: Maybe::Specified(Some(new_sc.into())), + storage_class: Maybe::Specified(Some(new_sc.clone())), ..Default::default() })), stream_cipher: Maybe::Specified(Some(new_algorithm.into())), @@ -963,7 +967,7 @@ mod tests { }; let result = base.reconfigure(reconfig); - prop_assert_eq!(result.default_stream_config.storage_class, Some(new_sc.into())); + prop_assert_eq!(result.default_stream_config.storage_class, Some(new_sc)); prop_assert_eq!(result.stream_cipher, Some(new_algorithm.into())); prop_assert_eq!(result.create_stream_on_append, new_on_append); } @@ -1057,4 +1061,67 @@ mod tests { "delete_on_empty.min_age should be None" ); } + + #[test] + fn stream_config_header_parses_and_validates() { + let header: StreamConfigHeader = + r#"{"retention_policy":{"age":3600},"delete_on_empty":{"min_age_secs":300}}"# + .parse() + .unwrap(); + assert_eq!( + header.0, + s2_common::config::OptionalStreamConfig { + retention_policy: Some(s2_common::config::RetentionPolicy::Age( + Duration::from_secs(3600) + )), + delete_on_empty: s2_common::config::OptionalDeleteOnEmptyConfig { + min_age: Some(Duration::from_secs(300)), + }, + ..Default::default() + } + ); + + for spaced in [ + r#"{ "retention_policy": { "age": 3600 }, "delete_on_empty": { "min_age_secs": 300 } }"#, + "{\t\"delete_on_empty\":\t{\"min_age_secs\":\t300},\t\"retention_policy\":\t{\"age\":\t3600}\t}", + " {\"retention_policy\":{\"age\":3600},\"delete_on_empty\":{\"min_age_secs\":300}} ", + ] { + let parsed: StreamConfigHeader = spaced.parse().unwrap(); + assert_eq!(parsed, header, "{spaced:?}"); + } + + let empty: StreamConfigHeader = "{}".parse().unwrap(); + assert_eq!(empty.0, Default::default()); + + let invalid_json = "not json".parse::().unwrap_err(); + assert!(invalid_json.to_string().contains("invalid JSON")); + + let invalid_age = + r#"{"retention_policy":{"age":0}}"#.parse::().unwrap_err(); + assert!( + invalid_age + .to_string() + .contains("age must be greater than 0 seconds"), + "{invalid_age}" + ); + } + + #[test] + fn stream_config_header_value_roundtrips() { + let config = StreamConfig { + storage_class: Some("express".into()), + retention_policy: Some(RetentionPolicy::Infinite(InfiniteRetention {})), + timestamping: Some(TimestampingConfig { + mode: Some(TimestampingMode::ClientRequire), + uncapped: Some(true), + }), + delete_on_empty: Some(DeleteOnEmptyConfig { min_age_secs: 60 }), + }; + let value = config.to_header_value(); + let parsed: StreamConfigHeader = value.to_str().unwrap().parse().unwrap(); + assert_eq!( + parsed.0, + s2_common::config::OptionalStreamConfig::try_from(config).unwrap() + ); + } } diff --git a/api/src/v1/location.rs b/api/src/v1/location.rs index 408486e7..aacb78a4 100644 --- a/api/src/v1/location.rs +++ b/api/src/v1/location.rs @@ -1,3 +1,4 @@ +use compact_str::CompactString; use s2_common::{self, location::LocationName}; use serde::{Deserialize, Serialize}; @@ -9,13 +10,31 @@ pub struct LocationInfo { pub name: LocationName, /// Location represents a private placement, limited by account. pub is_private: bool, + /// [Storage classes](https://s2.dev/docs/storage-classes) available to the account in this location. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "utoipa", schema(value_type = Option>))] + pub storage_classes: Option>, + /// Default [storage class](https://s2.dev/docs/storage-classes) for this location. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "utoipa", schema(value_type = Option))] + pub default_storage_class: Option, } impl From for LocationInfo { fn from(value: s2_common::location::LocationInfo) -> Self { - let s2_common::location::LocationInfo { name, is_private } = value; + let s2_common::location::LocationInfo { + name, + is_private, + storage_classes, + default_storage_class, + } = value; - Self { name, is_private } + Self { + name, + is_private, + storage_classes: Some(storage_classes), + default_storage_class: Some(default_storage_class), + } } } diff --git a/api/src/v1/stream/extract.rs b/api/src/v1/stream/extract.rs index 6d508d45..deae94fb 100644 --- a/api/src/v1/stream/extract.rs +++ b/api/src/v1/stream/extract.rs @@ -17,7 +17,7 @@ use crate::{ extract::{JsonExtractionRejection, ProtoRejection}, }, mime::JsonOrProto, - v1::stream::sse::LastEventId, + v1::{config::StreamConfigHeader, stream::sse::LastEventId}, }; #[derive(Debug, thiserror::Error)] @@ -54,6 +54,9 @@ where async fn from_request(req: Request, state: &S) -> Result { let content_type = crate::mime::content_type(req.headers()); let encryption_key = parse_header_opt::(req.headers())?; + let create_stream_config_patch = parse_header_opt::(req.headers())? + .map(|header| header.0) + .unwrap_or_default(); if content_type.as_ref().is_some_and(crate::mime::is_s2s_proto) { let response_compression = @@ -88,6 +91,7 @@ where return Ok(Self::S2s { encryption_key, + create_stream_config_patch, inputs: Box::pin(inputs), response_compression, }); @@ -117,6 +121,7 @@ where Ok(Self::Unary { encryption_key, + create_stream_config_patch, input, response_mime, }) @@ -132,12 +137,16 @@ where async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { let content_type = crate::mime::content_type(&parts.headers); let encryption_key = parse_header_opt::(&parts.headers)?; + let create_stream_config_patch = parse_header_opt::(&parts.headers)? + .map(|header| header.0) + .unwrap_or_default(); if content_type.as_ref().is_some_and(crate::mime::is_s2s_proto) { let response_compression = s2s::CompressionAlgorithm::from_accept_encoding(&parts.headers); return Ok(Self::S2s { encryption_key, + create_stream_config_patch, response_compression, }); } @@ -150,6 +159,7 @@ where let last_event_id = parse_header_opt::(&parts.headers)?; return Ok(Self::EventStream { encryption_key, + create_stream_config_patch, format, last_event_id, }); @@ -162,6 +172,7 @@ where Ok(Self::Unary { encryption_key, + create_stream_config_patch, format, response_mime, }) diff --git a/api/src/v1/stream/mod.rs b/api/src/v1/stream/mod.rs index 255879cd..2d8d66e8 100644 --- a/api/src/v1/stream/mod.rs +++ b/api/src/v1/stream/mod.rs @@ -11,6 +11,7 @@ use std::time::Duration; use futures_core::stream::BoxStream; use itertools::Itertools as _; use s2_common::{ + config::OptionalStreamConfig, encryption::EncryptionKey, record, stream::{StreamName, StreamNamePrefix, StreamNameStartAfter}, @@ -210,18 +211,24 @@ pub enum ReadRequest { /// Unary Unary { encryption_key: Option, + /// Parsed `s2-stream-config` header; empty if absent. + create_stream_config_patch: OptionalStreamConfig, format: Format, response_mime: JsonOrProto, }, /// Server-Sent Events streaming response EventStream { encryption_key: Option, + /// Parsed `s2-stream-config` header; empty if absent. + create_stream_config_patch: OptionalStreamConfig, format: Format, last_event_id: Option, }, /// S2S streaming response S2s { encryption_key: Option, + /// Parsed `s2-stream-config` header; empty if absent. + create_stream_config_patch: OptionalStreamConfig, response_compression: s2s::CompressionAlgorithm, }, } @@ -230,12 +237,16 @@ pub enum AppendRequest { /// Unary Unary { encryption_key: Option, + /// Parsed `s2-stream-config` header; empty if absent. + create_stream_config_patch: OptionalStreamConfig, input: s2_common::stream::AppendInput, response_mime: JsonOrProto, }, /// S2S bi-directional streaming S2s { encryption_key: Option, + /// Parsed `s2-stream-config` header; empty if absent. + create_stream_config_patch: OptionalStreamConfig, inputs: BoxStream<'static, Result>, response_compression: s2s::CompressionAlgorithm, }, @@ -246,21 +257,25 @@ impl std::fmt::Debug for AppendRequest { match self { AppendRequest::Unary { encryption_key, + create_stream_config_patch, input, response_mime: response, } => f .debug_struct("AppendRequest::Unary") .field("encryption_key", encryption_key) + .field("create_stream_config_patch", create_stream_config_patch) .field("input", input) .field("response", response) .finish(), AppendRequest::S2s { encryption_key, + create_stream_config_patch, response_compression, .. } => f .debug_struct("AppendRequest::S2s") .field("encryption_key", encryption_key) + .field("create_stream_config_patch", create_stream_config_patch) .field("response_compression", response_compression) .finish(), } diff --git a/api/src/v1/stream/s2s.rs b/api/src/v1/stream/s2s.rs index 517745d3..ae6c6304 100644 --- a/api/src/v1/stream/s2s.rs +++ b/api/src/v1/stream/s2s.rs @@ -71,7 +71,11 @@ impl CompressionAlgorithm { for header_value in headers.get_all(http::header::ACCEPT_ENCODING) { if let Ok(value) = header_value.to_str() { for encoding in value.split(',') { - let encoding = encoding.trim().split(';').next().unwrap_or("").trim(); + let mut parts = encoding.split(';'); + let encoding = parts.next().unwrap_or("").trim(); + if parts.any(is_zero_qvalue) { + continue; + } if encoding.eq_ignore_ascii_case("zstd") { return Self::Zstd; } else if encoding.eq_ignore_ascii_case("gzip") { @@ -84,6 +88,15 @@ impl CompressionAlgorithm { } } +/// Whether an `Accept-Encoding` parameter is a `q=0` weight, which marks the coding as +/// "not acceptable" (RFC 9110 §12.4.2). +fn is_zero_qvalue(param: &str) -> bool { + let Some((name, value)) = param.split_once('=') else { + return false; + }; + name.trim().eq_ignore_ascii_case("q") && value.trim().parse::().is_ok_and(|q| q == 0.0) +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct CompressedData { compression: CompressionAlgorithm, @@ -578,6 +591,26 @@ mod test { assert_eq!(algo, CompressionAlgorithm::Gzip); } + #[rstest::rstest] + #[case("zstd;q=0, gzip", CompressionAlgorithm::Gzip)] + #[case("zstd; q=0.0, gzip;q=0.5", CompressionAlgorithm::Gzip)] + #[case("gzip;q=0", CompressionAlgorithm::None)] + #[case("gzip;Q=0.000, zstd;q=0", CompressionAlgorithm::None)] + #[case("zstd;q=0.001", CompressionAlgorithm::Zstd)] + fn from_accept_encoding_skips_refused_codings( + #[case] accept_encoding: &'static str, + #[case] expected: CompressionAlgorithm, + ) { + let mut headers = http::HeaderMap::new(); + headers.insert( + http::header::ACCEPT_ENCODING, + HeaderValue::from_static(accept_encoding), + ); + + let algo = CompressionAlgorithm::from_accept_encoding(&headers); + assert_eq!(algo, expected); + } + #[test] fn from_accept_encoding_defaults_to_none() { let headers = http::HeaderMap::new(); diff --git a/api/src/v1/stream/sse.rs b/api/src/v1/stream/sse.rs index f6c5b955..7fad5d0f 100644 --- a/api/src/v1/stream/sse.rs +++ b/api/src/v1/stream/sse.rs @@ -9,8 +9,11 @@ static LAST_EVENT_ID_HEADER: http::HeaderName = http::HeaderName::from_static("l #[derive(Debug, Clone, Copy)] pub struct LastEventId { + /// Sequence number of the last delivered record. pub seq_num: u64, + /// Total records delivered. pub count: usize, + /// Total metered bytes delivered. pub bytes: usize, } diff --git a/charts/s2-lite-helm/Chart.yaml b/charts/s2-lite-helm/Chart.yaml index 35c7c06e..953a1674 100644 --- a/charts/s2-lite-helm/Chart.yaml +++ b/charts/s2-lite-helm/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: s2-lite-helm description: Self-hostable S2 streaming datastore using SlateDB on object storage type: application -version: 0.1.64 -appVersion: "0.42.8" +version: 0.1.71 +appVersion: "0.43.0" keywords: - s2 - streaming diff --git a/charts/s2-lite-helm/README.md b/charts/s2-lite-helm/README.md index e019e02e..ae02ba60 100644 --- a/charts/s2-lite-helm/README.md +++ b/charts/s2-lite-helm/README.md @@ -54,6 +54,40 @@ helm install my-s2-lite s2/s2-lite-helm \ Supports AWS S3, MinIO, Tigris, Cloudflare R2, and other S3-compatible services. +### Persistent volume + +```bash +helm install my-s2-lite s2/s2-lite-helm \ + --set persistentVolume.enabled=true \ + --set persistentVolume.size=20Gi +``` + +Stores the database on a `PersistentVolumeClaim` instead of a bucket. Set +`persistentVolume.existingClaim` to use your own claim. + +### Separate WAL storage + +The write-ahead log can live in a separate bucket or on its own persistent +volume (see [Storage](../../README.md#storage)). Requires `objectStorage` or +`persistentVolume`. + +```bash +# WAL on a persistent volume +helm install my-s2-lite s2/s2-lite-helm \ + --set objectStorage.enabled=true \ + --set objectStorage.bucket=my-bucket \ + --set walStorage.persistentVolume.enabled=true + +# WAL in a separate bucket +helm install my-s2-lite s2/s2-lite-helm \ + --set objectStorage.enabled=true \ + --set objectStorage.bucket=my-bucket \ + --set walStorage.bucket=my-wal-bucket +``` + +Keep the WAL location the same across upgrades; changing it does not migrate +WAL data and the server starts without an error. + ### TLS Configuration **Self-signed certificate (for dev/testing):** @@ -104,6 +138,11 @@ Common configurations: | `objectStorage.enabled` | Enable S3-compatible storage | `false` | | `objectStorage.bucket` | S3 bucket name | `""` | | `objectStorage.path` | Path prefix within bucket | `""` | +| `persistentVolume.enabled` | Store the database on a persistent volume | `false` | +| `persistentVolume.size` | Volume size | `10Gi` | +| `walStorage.bucket` | Separate S3 bucket for the WAL | `""` | +| `walStorage.persistentVolume.enabled` | Store the WAL on a persistent volume | `false` | +| `walStorage.persistentVolume.size` | WAL volume size | `10Gi` | | `metrics.serviceMonitor.enabled` | Enable Prometheus ServiceMonitor | `false` | ## Examples @@ -125,6 +164,21 @@ serviceAccount: helm install my-s2-lite s2/s2-lite-helm -f values.yaml ``` +### S3 with a local WAL volume + +```yaml +# values.yaml +objectStorage: + enabled: true + bucket: my-s3-bucket + +walStorage: + persistentVolume: + enabled: true + size: 20Gi + storageClass: gp3 +``` + ### Behind AWS Network Load Balancer ```yaml @@ -165,3 +219,14 @@ helm upgrade my-s2-lite s2/s2-lite-helm --version 0.1.0 helm uninstall my-s2-lite ``` +This deletes any `PersistentVolumeClaim` the chart created, and with it the data on the +volume. To keep the claim, annotate it before uninstalling: + +```yaml +persistentVolume: + annotations: + helm.sh/resource-policy: keep +``` + +Claims supplied via `existingClaim` are never deleted by the chart. + diff --git a/charts/s2-lite-helm/templates/NOTES.txt b/charts/s2-lite-helm/templates/NOTES.txt index 84b9755c..ed9e7eb8 100644 --- a/charts/s2-lite-helm/templates/NOTES.txt +++ b/charts/s2-lite-helm/templates/NOTES.txt @@ -11,7 +11,9 @@ s2-lite is now running with the following configuration: - Replicas: {{ .Values.replicaCount }} - Image: {{ include "s2-lite.image" . }} -{{- if and .Values.objectStorage.enabled .Values.objectStorage.bucket }} +{{- $storageMode := include "s2-lite.storage.mode" . }} +{{- $walMode := include "s2-lite.walStorage.mode" . }} +{{- if eq $storageMode "bucket" }} - Storage: S3-compatible ({{ .Values.objectStorage.bucket }}) {{- if .Values.objectStorage.path }} - Path: {{ .Values.objectStorage.path }} @@ -19,9 +21,16 @@ s2-lite is now running with the following configuration: {{- if .Values.objectStorage.endpoint }} - Endpoint: {{ .Values.objectStorage.endpoint }} {{- end }} +{{- else if eq $storageMode "volume" }} + - Storage: Persistent volume ({{ include "s2-lite.claimName" (dict "root" . "name" "data" "volume" .Values.persistentVolume) }} at {{ .Values.persistentVolume.mountPath }}) {{- else }} - Storage: In-memory (data lost on pod restart) {{- end }} +{{- if eq $walMode "bucket" }} + - WAL: S3-compatible ({{ .Values.walStorage.bucket }}) +{{- else if eq $walMode "volume" }} + - WAL: Persistent volume ({{ include "s2-lite.claimName" (dict "root" . "name" "wal" "volume" .Values.walStorage.persistentVolume) }} at {{ .Values.walStorage.persistentVolume.mountPath }}) +{{- end }} Get the application URL by running these commands: export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "s2-lite.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") diff --git a/charts/s2-lite-helm/templates/_helpers.tpl b/charts/s2-lite-helm/templates/_helpers.tpl index 62d8dd83..faf28de6 100644 --- a/charts/s2-lite-helm/templates/_helpers.tpl +++ b/charts/s2-lite-helm/templates/_helpers.tpl @@ -66,3 +66,52 @@ Create the image name {{- $tag := .Values.image.tag | default .Chart.AppVersion }} {{- printf "%s:%s" .Values.image.repository $tag }} {{- end }} + +{{/* +Main storage mode: "bucket", "volume", or "" for in-memory. +*/}} +{{- define "s2-lite.storage.mode" -}} +{{- if and .Values.objectStorage.enabled .Values.persistentVolume.enabled }} +{{- fail "objectStorage.enabled and persistentVolume.enabled are mutually exclusive" }} +{{- end }} +{{- if and .Values.objectStorage.enabled (not .Values.objectStorage.bucket) }} +{{- fail "objectStorage.enabled is true but objectStorage.bucket is not set" }} +{{- end }} +{{- if .Values.objectStorage.enabled }}bucket{{- else if .Values.persistentVolume.enabled }}volume{{- end }} +{{- end }} + +{{/* +WAL storage mode: "bucket", "volume", or "" when the WAL shares the main store. +*/}} +{{- define "s2-lite.walStorage.mode" -}} +{{- $bucket := .Values.walStorage.bucket }} +{{- $volume := .Values.walStorage.persistentVolume.enabled }} +{{- if and $bucket $volume }} +{{- fail "walStorage.bucket and walStorage.persistentVolume.enabled are mutually exclusive" }} +{{- end }} +{{- if and (or $bucket $volume) (not (include "s2-lite.storage.mode" .)) }} +{{- fail "walStorage requires objectStorage or persistentVolume" }} +{{- end }} +{{- if $bucket }}bucket{{- else if $volume }}volume{{- end }} +{{- end }} + +{{/* +Persistent volumes in use, keyed by volume name. Parse with fromYaml. +*/}} +{{- define "s2-lite.persistentVolumes" -}} +{{- $volumes := dict }} +{{- if eq (include "s2-lite.storage.mode" .) "volume" }} +{{- $_ := set $volumes "data" .Values.persistentVolume }} +{{- end }} +{{- if eq (include "s2-lite.walStorage.mode" .) "volume" }} +{{- $_ := set $volumes "wal" .Values.walStorage.persistentVolume }} +{{- end }} +{{- toYaml $volumes }} +{{- end }} + +{{/* +PersistentVolumeClaim name for a volume. Takes (dict "root" $ "name" "volume" ). +*/}} +{{- define "s2-lite.claimName" -}} +{{- .volume.existingClaim | default (printf "%s-%s" (include "s2-lite.fullname" .root) .name) }} +{{- end }} diff --git a/charts/s2-lite-helm/templates/deployment.yaml b/charts/s2-lite-helm/templates/deployment.yaml index 34bee45b..377dadaa 100644 --- a/charts/s2-lite-helm/templates/deployment.yaml +++ b/charts/s2-lite-helm/templates/deployment.yaml @@ -1,3 +1,6 @@ +{{- $storageMode := include "s2-lite.storage.mode" . }} +{{- $walMode := include "s2-lite.walStorage.mode" . }} +{{- $volumes := include "s2-lite.persistentVolumes" . | fromYaml }} apiVersion: apps/v1 kind: Deployment metadata: @@ -52,27 +55,42 @@ spec: {{- fail "tls.enabled is true but neither tls.cert/tls.key nor tls.selfSigned are configured" }} {{- end }} {{- end }} - {{- if .Values.objectStorage.enabled }} - {{- if not .Values.objectStorage.bucket }} - {{- fail "objectStorage.enabled is true but objectStorage.bucket is not set" }} - {{- end }} + {{- if eq $storageMode "bucket" }} - --bucket - {{ .Values.objectStorage.bucket | quote }} - {{- if .Values.objectStorage.path }} + {{- with .Values.objectStorage.path }} - --path - - {{ .Values.objectStorage.path | quote }} + - {{ . | quote }} {{- end }} + {{- else if eq $storageMode "volume" }} + - --local-root + - {{ .Values.persistentVolume.mountPath | quote }} + {{- end }} + {{- if eq $walMode "bucket" }} + - --wal-bucket + - {{ .Values.walStorage.bucket | quote }} + {{- else if eq $walMode "volume" }} + - --wal-local-root + - {{ .Values.walStorage.persistentVolume.mountPath | quote }} {{- end }} ports: - name: http containerPort: {{ .Values.service.targetPort }} protocol: TCP - {{- if or (and .Values.objectStorage.enabled .Values.objectStorage.endpoint) .Values.env }} + {{- if or (and .Values.objectStorage.enabled .Values.objectStorage.endpoint) .Values.walStorage.endpoint .Values.walStorage.region .Values.env }} env: {{- if and .Values.objectStorage.enabled .Values.objectStorage.endpoint }} - name: AWS_ENDPOINT_URL_S3 value: {{ .Values.objectStorage.endpoint | quote }} {{- end }} + {{- with .Values.walStorage.endpoint }} + - name: S2LITE_WAL_AWS_ENDPOINT_URL_S3 + value: {{ . | quote }} + {{- end }} + {{- with .Values.walStorage.region }} + - name: S2LITE_WAL_AWS_REGION + value: {{ . | quote }} + {{- end }} {{- with .Values.env }} {{- toYaml . | nindent 8 }} {{- end }} @@ -108,14 +126,27 @@ spec: {{- end }} resources: {{- toYaml .Values.resources | nindent 10 }} - {{- with .Values.volumeMounts }} + {{- if or $volumes .Values.volumeMounts }} volumeMounts: + {{- range $name, $volume := $volumes }} + - name: {{ $name }} + mountPath: {{ $volume.mountPath | quote }} + {{- end }} + {{- with .Values.volumeMounts }} {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.volumes }} + {{- end }} + {{- if or $volumes .Values.volumes }} volumes: + {{- range $name, $volume := $volumes }} + - name: {{ $name }} + persistentVolumeClaim: + claimName: {{ include "s2-lite.claimName" (dict "root" $ "name" $name "volume" $volume) }} + {{- end }} + {{- with .Values.volumes }} {{- toYaml . | nindent 6 }} {{- end }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/s2-lite-helm/templates/pvc.yaml b/charts/s2-lite-helm/templates/pvc.yaml new file mode 100644 index 00000000..b19594c1 --- /dev/null +++ b/charts/s2-lite-helm/templates/pvc.yaml @@ -0,0 +1,24 @@ +{{- range $name, $volume := include "s2-lite.persistentVolumes" . | fromYaml }} +{{- if not $volume.existingClaim }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "s2-lite.claimName" (dict "root" $ "name" $name "volume" $volume) }} + labels: + {{- include "s2-lite.labels" $ | nindent 4 }} + {{- with $volume.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + accessModes: + - ReadWriteOnce + {{- with $volume.storageClass }} + storageClassName: {{ . | quote }} + {{- end }} + resources: + requests: + storage: {{ $volume.size | quote }} +{{- end }} +{{- end }} diff --git a/charts/s2-lite-helm/values.yaml b/charts/s2-lite-helm/values.yaml index 19087c99..22336d60 100644 --- a/charts/s2-lite-helm/values.yaml +++ b/charts/s2-lite-helm/values.yaml @@ -102,7 +102,8 @@ ingress: # Storage Configuration # # By default, s2-lite runs in-memory (great for dev/testing). -# For persistent data, enable objectStorage with an S3-compatible bucket. +# For persistent data, enable objectStorage with an S3-compatible bucket or +# persistentVolume for a local disk. Enable at most one. # ------------------------------------------------------------------------------ objectStorage: @@ -118,6 +119,45 @@ objectStorage: # R2: "https://.r2.cloudflarestorage.com" endpoint: "" +# Persistent volume for the database (--local-root) +persistentVolume: + enabled: false + mountPath: /data + size: 10Gi + storageClass: "" + # Use an existing PersistentVolumeClaim instead of creating one + existingClaim: "" + # helm uninstall deletes the created claim; add helm.sh/resource-policy: keep to retain it + annotations: {} + +# ------------------------------------------------------------------------------ +# WAL Storage Configuration +# +# By default the write-ahead log (WAL) is stored with the database. Place it in +# a separate bucket or on its own persistent volume to isolate WAL latency from +# flushes and compaction. Requires objectStorage or persistentVolume; set at +# most one of bucket or persistentVolume.enabled. +# +# Keep the WAL location the same across upgrades: changing it does not migrate +# WAL data and the server starts without an error. +# ------------------------------------------------------------------------------ + +walStorage: + # S3 bucket for the WAL (--wal-bucket). Shares objectStorage's AWS + # configuration unless overridden below. For different credentials, set + # S2LITE_WAL_AWS_ACCESS_KEY_ID / S2LITE_WAL_AWS_SECRET_ACCESS_KEY via env. + bucket: "" + endpoint: "" + region: "" + # Persistent volume for the WAL (--wal-local-root) + persistentVolume: + enabled: false + mountPath: /wal + size: 10Gi + storageClass: "" + existingClaim: "" + annotations: {} + # ------------------------------------------------------------------------------ # Observability # ------------------------------------------------------------------------------ diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 9dfeac91..9004262c 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -2,6 +2,54 @@ All notable changes to this project will be documented in this file. +## [0.43.0] - 2026-09-25 + +### Features + +- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) + +### Bug Fixes + +- Show match-none token scopes as no access, not as wildcards ([#782](https://github.com/s2-streamstore/s2/issues/782)) + + + +## [0.42.14] - 2026-09-24 + + + +## [0.42.13] - 2026-09-22 + +### Features + +- Support a separate WAL object store ([#766](https://github.com/s2-streamstore/s2/issues/766)) + + + +## [0.42.12] - 2026-09-16 + +### Miscellaneous Tasks + +- Update Cargo.lock dependencies + + + +## [0.42.11] - 2026-09-11 + +### Features + +- `s2-stream-config` header for auto-created streams ([#718](https://github.com/s2-streamstore/s2/issues/718)) + + + +## [0.42.10] - 2026-09-11 + + + +## [0.42.9] - 2026-09-10 + + + ## [0.42.8] - 2026-09-01 ### Bug Fixes diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 64dfddeb..b5284ce4 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-cli" -version = "0.42.8" +version = "0.43.0" description = "CLI for S2" edition.workspace = true license.workspace = true @@ -21,6 +21,7 @@ bytes = { workspace = true } clap = { workspace = true, features = ["derive"] } color-print = { workspace = true } colored = { workspace = true } +compact_str = { workspace = true, features = ["serde"] } config = { workspace = true } dirs = { workspace = true } fs2 = { workspace = true } diff --git a/cli/schema.json b/cli/schema.json index c6f9c997..c785bd21 100644 --- a/cli/schema.json +++ b/cli/schema.json @@ -87,14 +87,10 @@ "type": "object", "properties": { "storage_class": { - "description": "Storage class for recent writes.", - "anyOf": [ - { - "$ref": "#/$defs/StorageClass" - }, - { - "type": "null" - } + "description": "[Storage class](https://s2.dev/docs/storage-classes) for recent writes.", + "type": [ + "string", + "null" ], "default": null }, @@ -134,14 +130,6 @@ }, "additionalProperties": false }, - "StorageClass": { - "description": "Storage class for recent writes.", - "type": "string", - "enum": [ - "standard", - "express" - ] - }, "RetentionPolicy": { "description": "Retain records unless explicitly trimmed (\"infinite\"), or automatically trim records older than the given duration (e.g. \"7days\", \"1week\"). Age durations must be greater than 0 seconds.", "type": "string", diff --git a/cli/src/apply.rs b/cli/src/apply.rs index 7235508d..372b2d0e 100644 --- a/cli/src/apply.rs +++ b/cli/src/apply.rs @@ -3,88 +3,19 @@ use std::path::Path; use colored::Colorize; +use miette::IntoDiagnostic; use s2_common::{ basin::BasinName, - config::{ - BasinConfig, OptionalDeleteOnEmptyConfig, OptionalStreamConfig, OptionalTimestampingConfig, - RetentionPolicy, StorageClass, StreamConfig, TimestampingMode, - }, + config::{RetentionPolicy, StreamConfig, TimestampingMode}, encryption::EncryptionAlgorithm, stream::StreamName, }; -use s2_sdk::error::{ErrorCode, RequestError}; +use s2_sdk::{ + error::{ErrorCode, RequestError, StatusCode}, + types::BasinConfig, +}; -fn basin_config_from_sdk(config: s2_sdk::types::BasinConfig) -> BasinConfig { - BasinConfig { - default_stream_config: config - .default_stream_config - .map(optional_stream_config_from_sdk) - .unwrap_or_default(), - stream_cipher: config.stream_cipher, - create_stream_on_append: config.create_stream_on_append, - create_stream_on_read: config.create_stream_on_read, - } -} - -fn stream_config_from_sdk(config: s2_sdk::types::StreamConfig) -> StreamConfig { - optional_stream_config_from_sdk(config).into() -} - -fn optional_stream_config_from_sdk(config: s2_sdk::types::StreamConfig) -> OptionalStreamConfig { - OptionalStreamConfig { - storage_class: config.storage_class.map(storage_class_from_sdk), - retention_policy: config.retention_policy.map(retention_policy_from_sdk), - timestamping: config - .timestamping - .map(timestamping_from_sdk) - .unwrap_or_default(), - delete_on_empty: config - .delete_on_empty - .map(delete_on_empty_from_sdk) - .unwrap_or_default(), - } -} - -fn storage_class_from_sdk(storage_class: s2_sdk::types::StorageClass) -> StorageClass { - match storage_class { - s2_sdk::types::StorageClass::Standard => StorageClass::Standard, - s2_sdk::types::StorageClass::Express => StorageClass::Express, - } -} - -fn retention_policy_from_sdk(retention_policy: s2_sdk::types::RetentionPolicy) -> RetentionPolicy { - match retention_policy { - s2_sdk::types::RetentionPolicy::Age(secs) => { - RetentionPolicy::Age(std::time::Duration::from_secs(secs)) - } - s2_sdk::types::RetentionPolicy::Infinite => RetentionPolicy::Infinite(), - } -} - -fn timestamping_from_sdk( - timestamping: s2_sdk::types::TimestampingConfig, -) -> OptionalTimestampingConfig { - OptionalTimestampingConfig { - mode: timestamping.mode.map(timestamping_mode_from_sdk), - uncapped: timestamping.uncapped, - } -} - -fn timestamping_mode_from_sdk(mode: s2_sdk::types::TimestampingMode) -> TimestampingMode { - match mode { - s2_sdk::types::TimestampingMode::ClientPrefer => TimestampingMode::ClientPrefer, - s2_sdk::types::TimestampingMode::ClientRequire => TimestampingMode::ClientRequire, - s2_sdk::types::TimestampingMode::Arrival => TimestampingMode::Arrival, - } -} - -fn delete_on_empty_from_sdk( - delete_on_empty: s2_sdk::types::DeleteOnEmptyConfig, -) -> OptionalDeleteOnEmptyConfig { - OptionalDeleteOnEmptyConfig { - min_age: Some(std::time::Duration::from_secs(delete_on_empty.min_age_secs)), - } -} +use crate::types::resolve_stream_config; fn basin_config_to_sdk(config: s2_resource_spec::BasinConfig) -> s2_sdk::types::BasinConfig { let mut sdk_config = s2_sdk::types::BasinConfig::new(); @@ -107,7 +38,7 @@ fn basin_config_to_sdk(config: s2_resource_spec::BasinConfig) -> s2_sdk::types:: fn stream_config_to_sdk(config: s2_resource_spec::StreamConfig) -> s2_sdk::types::StreamConfig { let mut sdk_config = s2_sdk::types::StreamConfig::new(); if let Some(storage_class) = config.storage_class { - sdk_config = sdk_config.with_storage_class(storage_class_to_sdk(storage_class)); + sdk_config = sdk_config.with_storage_class(storage_class); } if let Some(retention_policy) = config.retention_policy { sdk_config = sdk_config.with_retention_policy(retention_policy_to_sdk(retention_policy)); @@ -121,15 +52,6 @@ fn stream_config_to_sdk(config: s2_resource_spec::StreamConfig) -> s2_sdk::types sdk_config } -fn storage_class_to_sdk( - storage_class: s2_resource_spec::StorageClass, -) -> s2_sdk::types::StorageClass { - match storage_class { - s2_resource_spec::StorageClass::Standard => s2_sdk::types::StorageClass::Standard, - s2_resource_spec::StorageClass::Express => s2_sdk::types::StorageClass::Express, - } -} - fn retention_policy_to_sdk( retention_policy: s2_resource_spec::RetentionPolicy, ) -> s2_sdk::types::RetentionPolicy { @@ -290,10 +212,26 @@ async fn apply_stream( enum ResourceAction { Create, - Ensure(Vec), + Ensure { + diffs: Vec, + storage_class_unresolved: bool, + }, Unchanged, } +impl ResourceAction { + fn from_diff(diffs: Vec, storage_class_unresolved: bool) -> Self { + if diffs.is_empty() && !storage_class_unresolved { + Self::Unchanged + } else { + Self::Ensure { + diffs, + storage_class_unresolved, + } + } + } +} + struct FieldDiff { field: &'static str, old: String, @@ -307,13 +245,6 @@ fn is_not_found_error(error: &RequestError) -> bool { .is_some_and(|code| matches!(code, ErrorCode::BasinNotFound | ErrorCode::StreamNotFound)) } -fn format_storage_class(sc: StorageClass) -> &'static str { - match sc { - StorageClass::Standard => "standard", - StorageClass::Express => "express", - } -} - fn format_retention_policy(rp: RetentionPolicy) -> String { match rp { RetentionPolicy::Age(age) => humantime::format_duration(age).to_string(), @@ -340,7 +271,10 @@ fn default_stream_config_field(field: &'static str) -> &'static str { } } -fn diff_basin_config(existing: &BasinConfig, desired: &BasinConfig) -> Vec { +fn diff_basin_config( + existing: &BasinConfig, + desired: &BasinConfig, +) -> miette::Result> { let mut diffs = Vec::new(); if existing.stream_cipher != desired.stream_cipher { @@ -375,8 +309,24 @@ fn diff_basin_config(existing: &BasinConfig, desired: &BasinConfig) -> Vec Vec Vec { let mut diffs = Vec::new(); - if existing.storage_class != desired.storage_class { + if let Some(storage_class) = &desired.storage_class + && existing.storage_class.as_ref() != Some(storage_class) + { diffs.push(FieldDiff { field: "storage_class", - old: format_storage_class(existing.storage_class).to_string(), - new: format_storage_class(desired.storage_class).to_string(), + old: existing + .storage_class + .as_deref() + .unwrap_or("unspecified") + .to_owned(), + new: storage_class.to_string(), }); } @@ -478,7 +434,7 @@ fn spec_stream_fields(spec: &s2_resource_spec::StreamConfig) -> Vec { fields.push(FieldDiff { field: "storage_class", old: String::new(), - new: format_storage_class(sc.clone().into()).to_string(), + new: sc.to_string(), }); } if let Some(ref rp) = spec.retention_policy { @@ -522,11 +478,18 @@ fn print_basin_result(basin: &str, action: &ResourceAction) { ResourceAction::Create => { println!("{}", format!("+ basin {basin}").green().bold()); } - ResourceAction::Ensure(diffs) => { - println!("{}", format!("~ basin {basin}").yellow().bold()); + ResourceAction::Ensure { + diffs, + storage_class_unresolved, + } => { + let marker = if diffs.is_empty() { "?" } else { "~" }; + println!("{}", format!("{marker} basin {basin}").yellow().bold()); for diff in diffs { println!(" {}: {} → {}", diff.field, diff.old.dimmed(), diff.new); } + if *storage_class_unresolved { + println!(" default_stream_config.storage_class: resolved at apply"); + } } ResourceAction::Unchanged => { println!("{}", format!("= basin {basin}").dimmed()); @@ -539,11 +502,23 @@ fn print_stream_result(basin: &str, stream: &str, action: &ResourceAction) { ResourceAction::Create => { println!("{}", format!(" + stream {basin}/{stream}").green().bold()); } - ResourceAction::Ensure(diffs) => { - println!("{}", format!(" ~ stream {basin}/{stream}").yellow().bold()); + ResourceAction::Ensure { + diffs, + storage_class_unresolved, + } => { + let marker = if diffs.is_empty() { "?" } else { "~" }; + println!( + "{}", + format!(" {marker} stream {basin}/{stream}") + .yellow() + .bold() + ); for diff in diffs { println!(" {}: {} → {}", diff.field, diff.old.dimmed(), diff.new); } + if *storage_class_unresolved { + println!(" storage_class: resolved at apply"); + } } ResourceAction::Unchanged => { println!("{}", format!(" = stream {basin}/{stream}").dimmed()); @@ -572,24 +547,55 @@ fn print_stream_create(basin: &str, stream: &str, spec: &Option miette::Result<()> { validate(&spec)?; + let needs_location_default = spec.basins.iter().any(|basin| { + basin + .config + .as_ref() + .and_then(|config| config.default_stream_config.as_ref()) + .and_then(|config| config.storage_class.as_ref()) + .is_none() + }); + let default_storage_class = if needs_location_default { + match s2.get_default_location().await { + Ok(location) => location.default_storage_class, + Err(error) + if error.server_error().is_some_and(|error| { + matches!( + error.known_code(), + Some(ErrorCode::NotImplemented | ErrorCode::PermissionDenied) + ) || error.status == StatusCode::NOT_FOUND + }) => + { + None + } + Err(error) => { + return Err(miette::miette!( + "failed to get default storage class: {error}" + )); + } + } + } else { + None + }; + for basin_spec in spec.basins { - let desired_basin_config = basin_spec + let mut desired_basin_config = basin_spec .config .clone() - .map(BasinConfig::from) + .map(basin_config_to_sdk) .unwrap_or_default(); - let desired_basin_default_stream_config = - desired_basin_config.default_stream_config.clone(); + let desired_basin_defaults = desired_basin_config + .default_stream_config + .get_or_insert_default(); + if desired_basin_defaults.storage_class.is_none() { + desired_basin_defaults.storage_class = default_storage_class.clone(); + } + let desired_basin_defaults = desired_basin_defaults.clone(); let basin_action = match s2.get_basin_config(basin_spec.name.clone()).await { Ok(existing) => { - let existing = basin_config_from_sdk(existing); - let diffs = diff_basin_config(&existing, &desired_basin_config); - if diffs.is_empty() { - ResourceAction::Unchanged - } else { - ResourceAction::Ensure(diffs) - } + let diffs = diff_basin_config(&existing, &desired_basin_config)?; + ResourceAction::from_diff(diffs, desired_basin_defaults.storage_class.is_none()) } Err(e) if is_not_found_error(&e) => ResourceAction::Create, Err(e) => { @@ -618,19 +624,20 @@ pub async fn dry_run(s2: &s2_sdk::S2, spec: s2_resource_spec::Resources) -> miet .await { Ok(existing) => { - let existing = stream_config_from_sdk(existing); - let desired_stream_config = stream_spec - .config - .clone() - .map(OptionalStreamConfig::from) - .unwrap_or_default() - .merge(desired_basin_default_stream_config.clone()); + let existing = resolve_stream_config(existing.into(), Default::default()) + .into_diagnostic()?; + let desired_stream_config = resolve_stream_config( + stream_spec + .config + .clone() + .map(stream_config_to_sdk) + .unwrap_or_default() + .into(), + desired_basin_defaults.clone().into(), + ) + .into_diagnostic()?; let diffs = diff_stream_configs(&existing, &desired_stream_config); - if diffs.is_empty() { - ResourceAction::Unchanged - } else { - ResourceAction::Ensure(diffs) - } + ResourceAction::from_diff(diffs, desired_stream_config.storage_class.is_none()) } Err(e) if is_not_found_error(&e) => ResourceAction::Create, Err(e) => { diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 74981c1a..30f2baf7 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -1,6 +1,7 @@ use std::{num::NonZeroU64, path::PathBuf, time::Duration}; use clap::{Args, Parser, Subcommand, ValueEnum, builder::styling}; +use compact_str::CompactString; use s2_sdk::types::{ AccessTokenId, AccessTokenIdPrefix, AccessTokenIdStartAfter, BasinName, BasinNamePrefix, BasinNameStartAfter, EncryptionAlgorithm, EncryptionKey, FencingToken, StreamName, @@ -14,7 +15,7 @@ use crate::{ }, types::{ BasinConfig, Interval, LocationName, Operation, PermittedOperationGroups, - S2BasinAndMaybeStreamUri, S2BasinAndStreamUri, S2BasinUri, StorageClass, StreamConfig, + S2BasinAndMaybeStreamUri, S2BasinAndStreamUri, S2BasinUri, StreamConfig, }, }; @@ -191,6 +192,7 @@ pub enum Command { /// `+` create /// `~` ensure /// `=` unchanged + /// `?` storage-class default resolved at apply /// /// For IDE validation/autocomplete, add `$schema` at the top of each spec file: /// {"$schema":"https://raw.githubusercontent.com/s2-streamstore/s2/main/cli/schema.json","basins":[]} @@ -693,6 +695,15 @@ pub struct AppendArgs { #[command(flatten)] pub encryption_key: EncryptionKeyArgs, + + /// Stream configuration to apply if the stream is created on append. + /// Unset fields inherit the basin's default stream configuration. + /// Ignored if the stream already exists. + #[command( + flatten, + next_help_heading = "Stream configuration (applied only if the stream is created on append)" + )] + pub stream_config: StreamConfig, } #[derive(Args, Debug, Clone, Default)] @@ -771,6 +782,15 @@ pub struct ReadArgs { #[command(flatten)] pub encryption_key: EncryptionKeyArgs, + + /// Stream configuration to apply if the stream is created on read. + /// Unset fields inherit the basin's default stream configuration. + /// Ignored if the stream already exists. + #[command( + flatten, + next_help_heading = "Stream configuration (applied only if the stream is created on read)" + )] + pub stream_config: StreamConfig, } #[derive(Args, Debug)] @@ -816,6 +836,7 @@ pub struct ApplyArgs { /// `+` create /// `~` ensure /// `=` unchanged + /// `?` storage-class default resolved at apply #[arg(long)] pub dry_run: bool, /// Print the JSON Schema for the spec file format to stdout. @@ -830,7 +851,7 @@ pub struct BenchArgs { /// Storage class for the test stream. Uses basin default if not specified. #[arg(short = 'c', long)] - pub storage_class: Option, + pub storage_class: Option, /// Total metered record size in bytes (includes headers and overhead). #[arg( diff --git a/cli/src/diff.rs b/cli/src/diff.rs index 75f612a4..6ea1ffe3 100644 --- a/cli/src/diff.rs +++ b/cli/src/diff.rs @@ -1,16 +1,12 @@ -//! Diffing over typed presentation views of Basins, Streams, and Access Tokens. -//! -//! - Configs are fetched as API DTOs and converted into `s2_common` domain types via the existing -//! `TryFrom` conversions, which also resolve omitted fields to their effective defaults (e.g. an -//! unset `storage_class` becomes `express`). -//! -//! - Domain values are rendered into `*View` structs and serialized to JSON, which is diffed -//! field-by-field. Views own all presentation: compact exact durations (`"7d"`, `"1h"`), -//! wire-format enum names, canonically ordered operation sets. +//! Diffing over typed presentation views of basins, streams, and access tokens. use colored::Colorize; +use compact_str::CompactString; use s2_api::v1::access::AccessTokenInfo as ApiAccessTokenInfo; -use s2_common::access::{PermittedOperationGroups, ReadWritePermissions, ResourceSet}; +use s2_common::{ + access::{PermittedOperationGroups, ReadWritePermissions, ResourceSet}, + config::StreamConfig, +}; use s2_sdk::types::AccessTokenId; use serde::Serialize; use serde_json::{Map, Value}; @@ -19,7 +15,7 @@ use crate::{ cli::{DiffArgs, DiffOutput, DiffResourceKind}, error::CliError, ops, - types::{DiffResource, S2BasinAndStreamUri, S2BasinUri}, + types::{DiffResource, S2BasinAndStreamUri, S2BasinUri, resolve_stream_config}, }; #[derive(Debug, Serialize)] @@ -298,7 +294,7 @@ fn resource_kind(resource: &DiffResource) -> DiffResourceKind { } } -/// Presentation of an effective (fully resolved) stream configuration. +/// Presentation of a stream configuration for comparison. /// /// - Field names and nesting mirror the API wire format so that `--output json` paths line up with /// API responses. @@ -306,7 +302,7 @@ fn resource_kind(resource: &DiffResource) -> DiffResourceKind { /// duration. #[derive(Serialize)] struct StreamConfigView { - storage_class: String, + storage_class: Option, retention_policy: String, timestamping: TimestampingConfigView, delete_on_empty: DeleteOnEmptyConfigView, @@ -323,9 +319,9 @@ struct DeleteOnEmptyConfigView { min_age: String, } -impl From for StreamConfigView { - fn from(config: s2_common::config::StreamConfig) -> Self { - let s2_common::config::StreamConfig { +impl From for StreamConfigView { + fn from(config: StreamConfig) -> Self { + let StreamConfig { storage_class, retention_policy, timestamping, @@ -334,7 +330,7 @@ impl From for StreamConfigView { let s2_common::config::TimestampingConfig { mode, uncapped } = timestamping; Self { - storage_class: wire_name(s2_api::v1::config::StorageClass::from(storage_class)), + storage_class, retention_policy: match retention_policy { s2_common::config::RetentionPolicy::Age(age) => compact_duration(age), s2_common::config::RetentionPolicy::Infinite() => "infinite".to_owned(), @@ -359,25 +355,30 @@ struct BasinConfigView { create_stream_on_read: bool, } -impl From for BasinConfigView { - fn from(config: s2_common::config::BasinConfig) -> Self { - let s2_common::config::BasinConfig { +impl TryFrom for BasinConfigView { + type Error = s2_common::ValidationError; + + fn try_from(config: s2_api::v1::config::BasinConfig) -> Result { + let s2_api::v1::config::BasinConfig { default_stream_config, stream_cipher, create_stream_on_append, create_stream_on_read, } = config; - Self { - default_stream_config: s2_common::config::StreamConfig::from(default_stream_config) - .into(), + Ok(Self { + default_stream_config: resolve_stream_config( + default_stream_config.unwrap_or_default(), + Default::default(), + )? + .into(), stream_cipher: match stream_cipher { None => "none".to_owned(), - Some(cipher) => wire_name(s2_api::v1::config::EncryptionAlgorithm::from(cipher)), + Some(cipher) => wire_name(cipher), }, create_stream_on_append, create_stream_on_read, - } + }) } } @@ -464,20 +465,16 @@ fn permissions_view(permissions: ReadWritePermissions) -> String { } fn basin_view(config: s2_api::v1::config::BasinConfig) -> Result { - let config: s2_common::config::BasinConfig = config.try_into()?; - Ok(serde_json::to_value(BasinConfigView::from(config))?) + Ok(serde_json::to_value(BasinConfigView::try_from(config)?)?) } /// Renders a basin's effective stream defaults for comparison against a concrete stream. fn basin_stream_defaults_view(config: s2_api::v1::config::BasinConfig) -> Result { - let config: s2_common::config::BasinConfig = config.try_into()?; - let defaults = s2_common::config::StreamConfig::from(config.default_stream_config); - Ok(serde_json::to_value(StreamConfigView::from(defaults))?) + stream_view(config.default_stream_config.unwrap_or_default()) } fn stream_view(config: s2_api::v1::config::StreamConfig) -> Result { - let config: s2_common::config::OptionalStreamConfig = config.try_into()?; - let config = s2_common::config::StreamConfig::from(config); + let config = resolve_stream_config(config, Default::default())?; Ok(serde_json::to_value(StreamConfigView::from(config))?) } @@ -741,7 +738,7 @@ mod tests { assert_eq!( view, json!({ - "storage_class": "express", + "storage_class": null, "retention_policy": "2d", "timestamping": {"mode": "client-prefer", "uncapped": false}, "delete_on_empty": {"min_age": "1h"} @@ -773,7 +770,7 @@ mod tests { fn omitted_and_explicit_defaults_render_identically() { let omitted = serde_json::from_value(json!({})).expect("empty config deserializes"); let explicit = serde_json::from_value(json!({ - "storage_class": "express", + "storage_class": null, "retention_policy": {"age": 604800}, "timestamping": {"mode": "client-prefer", "uncapped": false}, "delete_on_empty": {"min_age_secs": 0} @@ -802,7 +799,7 @@ mod tests { view, json!({ "default_stream_config": { - "storage_class": "express", + "storage_class": null, "retention_policy": "7d", "timestamping": {"mode": "client-prefer", "uncapped": false}, "delete_on_empty": {"min_age": "0s"} diff --git a/cli/src/main.rs b/cli/src/main.rs index cc809202..7c23f771 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -42,7 +42,7 @@ use s2_sdk::{ S2, types::{ AppendRetryPolicy, CreateStreamInput, DeleteOnEmptyConfig, DeleteStreamInput, - EncryptionKey, MeteredBytes, Metric, RetentionPolicy, RetryConfig, + EncryptionKey, LocationInfo, MeteredBytes, Metric, RetentionPolicy, RetryConfig, StreamConfig as SdkStreamConfig, StreamName, TimestampingConfig, TimestampingMode, }, }; @@ -531,13 +531,13 @@ async fn run(cli: Cli) -> Result { Command::ListLocations => { let locations = ops::list_locations(&s2).await?; for location_info in locations { - print_location_listing(location_info.name.to_string(), location_info.is_private); + print_location_listing(&location_info); } } Command::GetDefaultLocation => { let location = ops::get_default_location(&s2).await?; - print_location_listing(location.name.to_string(), location.is_private); + print_location_listing(&location); } Command::SetDefaultLocation { location } => { @@ -549,7 +549,7 @@ async fn run(cli: Cli) -> Result { .green() .bold() ); - print_location_listing(location.name.to_string(), location.is_private); + print_location_listing(&location); } Command::GetAccountMetrics(args) => { @@ -657,9 +657,13 @@ async fn run(cli: Cli) -> Result { record_stream, args.uri, encryption_key.as_ref(), - args.fencing_token, - args.match_seq_num, - *args.linger, + ops::AppendOptions { + fencing_token: args.fencing_token, + match_seq_num: args.match_seq_num, + linger: *args.linger, + stream_config: (!args.stream_config.is_empty()) + .then(|| args.stream_config.into()), + }, ); let mut acks = std::pin::pin!(acks); let mut last_printed_batch_end: Option = None; @@ -848,7 +852,7 @@ async fn run(cli: Cli) -> Result { .with_mode(TimestampingMode::ClientRequire) .with_uncapped(true), ); - stream_config.storage_class = args.storage_class.map(Into::into); + stream_config.storage_class = args.storage_class; let s2 = S2::new(sdk_config.clone().with_retry( RetryConfig::new().with_append_retry_policy(AppendRetryPolicy::NoSideEffects), @@ -917,9 +921,20 @@ fn print_basin_listing(name: String, location: Option<&str>, is_deleting: bool) } } -fn print_location_listing(name: String, is_private: bool) { - let visibility = format_location_visibility(is_private); - println!("{name} {visibility}"); +fn print_location_listing(location: &LocationInfo) { + let visibility = format_location_visibility(location.is_private); + println!("{} {visibility}", location.name); + if let Some(storage_classes) = &location.storage_classes { + let classes = if storage_classes.is_empty() { + "none".to_owned() + } else { + storage_classes.join(", ") + }; + println!(" storage classes: {classes}"); + } + if let Some(default_storage_class) = &location.default_storage_class { + println!(" default storage class: {default_storage_class}"); + } } fn format_location_visibility(is_private: bool) -> colored::ColoredString { diff --git a/cli/src/ops.rs b/cli/src/ops.rs index 7ae3218d..468d6ab0 100644 --- a/cli/src/ops.rs +++ b/cli/src/ops.rs @@ -570,23 +570,32 @@ pub async fn read( stop = stop.with_until(..until); } + let mut input = ReadInput::new().with_start(start).with_stop(stop); + if !args.stream_config.is_empty() { + input = input.with_stream_config(args.stream_config.clone().into()); + } + stream - .read_session( - ReadInput::new().with_start(start).with_stop(stop), - ReadSessionConfig::default(), - ) + .read_session(input, ReadSessionConfig::default()) .await .map_err(|e| CliError::op(OpKind::Read, e)) } +/// Options controlling how records are appended. +pub struct AppendOptions { + pub fencing_token: Option, + pub match_seq_num: Option, + pub linger: Duration, + /// Stream configuration to apply if the stream is created on append. + pub stream_config: Option, +} + pub fn append<'a, S, E>( s2: &'a S2, records: S, uri: S2BasinAndStreamUri, encryption_key: Option<&'a EncryptionKey>, - fencing_token: Option, - match_seq_num: Option, - linger: Duration, + options: AppendOptions, ) -> impl Stream> + Send + 'a where S: Stream> + Send + Unpin + 'a, @@ -594,12 +603,15 @@ where { let stream = stream_with_encryption(s2, uri, encryption_key); - let batching_config = BatchingConfig::new().with_linger(linger); + let batching_config = BatchingConfig::new().with_linger(options.linger); let mut producer_config = ProducerConfig::new().with_batching(batching_config); - if let Some(ft) = fencing_token { + if let Some(config) = options.stream_config { + producer_config = producer_config.with_stream_config(config); + } + if let Some(ft) = options.fencing_token { producer_config = producer_config.with_fencing_token(ft); } - if let Some(seq) = match_seq_num { + if let Some(seq) = options.match_seq_num { producer_config = producer_config.with_match_seq_num(seq); } diff --git a/cli/src/types.rs b/cli/src/types.rs index 699a04b9..d6187cbc 100644 --- a/cli/src/types.rs +++ b/cli/src/types.rs @@ -2,6 +2,7 @@ use std::{str::FromStr, time::Duration}; use clap::{Args, Parser, ValueEnum}; use colored::Colorize; +use compact_str::CompactString; use s2_sdk::{ self as sdk, types::{ @@ -174,7 +175,7 @@ pub struct BasinConfig { pub struct StreamConfig { #[arg(long)] /// Storage class for a stream. - pub storage_class: Option, + pub storage_class: Option, #[arg(long, help("Example: 1d, 1w, 1y"))] /// Retention policy for a stream. pub retention_policy: Option, @@ -201,15 +202,16 @@ impl StreamConfig { } } -pub use sdk::types::LocationName; - -#[derive(ValueEnum, Debug, Clone, Serialize)] -#[serde(rename_all = "kebab-case")] -pub enum StorageClass { - Standard, - Express, +pub fn resolve_stream_config( + config: s2_api::v1::config::StreamConfig, + basin_defaults: s2_api::v1::config::StreamConfig, +) -> Result { + let config: s2_common::config::OptionalStreamConfig = config.try_into()?; + Ok(config.merge(basin_defaults.try_into()?)) } +pub use sdk::types::LocationName; + #[derive(ValueEnum, Debug, Clone, Serialize)] #[serde(rename_all = "kebab-case")] pub enum TimestampingMode { @@ -302,7 +304,7 @@ impl From for sdk::types::StreamConfig { fn from(config: StreamConfig) -> Self { let mut stream_config = sdk::types::StreamConfig::new(); if let Some(storage_class) = config.storage_class { - stream_config = stream_config.with_storage_class(storage_class.into()); + stream_config = stream_config.with_storage_class(storage_class); } if let Some(retention_policy) = config.retention_policy { stream_config = stream_config.with_retention_policy(retention_policy.into()); @@ -317,24 +319,6 @@ impl From for sdk::types::StreamConfig { } } -impl From for sdk::types::StorageClass { - fn from(class: StorageClass) -> Self { - match class { - StorageClass::Standard => sdk::types::StorageClass::Standard, - StorageClass::Express => sdk::types::StorageClass::Express, - } - } -} - -impl From for StorageClass { - fn from(class: sdk::types::StorageClass) -> Self { - match class { - sdk::types::StorageClass::Standard => StorageClass::Standard, - sdk::types::StorageClass::Express => StorageClass::Express, - } - } -} - impl From for sdk::types::TimestampingMode { fn from(mode: TimestampingMode) -> Self { match mode { @@ -414,7 +398,7 @@ impl From for BasinConfig { impl From for StreamConfig { fn from(config: sdk::types::StreamConfig) -> Self { StreamConfig { - storage_class: config.storage_class.map(Into::into), + storage_class: config.storage_class, retention_policy: config.retention_policy.map(Into::into), timestamping: config.timestamping.map(Into::into), delete_on_empty: config.delete_on_empty.map(Into::into), @@ -426,7 +410,7 @@ impl From for sdk::types::StreamReconfiguration { fn from(config: StreamConfig) -> Self { let mut reconfig = sdk::types::StreamReconfiguration::new(); if let Some(storage_class) = config.storage_class { - reconfig = reconfig.with_storage_class(storage_class.into()); + reconfig = reconfig.with_storage_class(storage_class); } if let Some(retention_policy) = config.retention_policy { reconfig = reconfig.with_retention_policy(retention_policy.into()); @@ -497,32 +481,35 @@ where serializer.serialize_str(&humantime::format_duration(*value).to_string()) } -impl From for BasinMatcher { - fn from(matcher: sdk::types::BasinMatcher) -> Self { +impl BasinMatcher { + /// Converts an SDK matcher, returning `None` for a matcher that matches no resources. + fn from_sdk(matcher: sdk::types::BasinMatcher) -> Option { match matcher { - sdk::types::BasinMatcher::Exact(v) => BasinMatcher::Exact(v), - sdk::types::BasinMatcher::Prefix(v) => BasinMatcher::Prefix(v), - sdk::types::BasinMatcher::None => BasinMatcher::Prefix(Default::default()), + sdk::types::BasinMatcher::Exact(v) => Some(BasinMatcher::Exact(v)), + sdk::types::BasinMatcher::Prefix(v) => Some(BasinMatcher::Prefix(v)), + sdk::types::BasinMatcher::None => None, } } } -impl From for StreamMatcher { - fn from(matcher: sdk::types::StreamMatcher) -> Self { +impl StreamMatcher { + /// Converts an SDK matcher, returning `None` for a matcher that matches no resources. + fn from_sdk(matcher: sdk::types::StreamMatcher) -> Option { match matcher { - sdk::types::StreamMatcher::Exact(v) => StreamMatcher::Exact(v), - sdk::types::StreamMatcher::Prefix(v) => StreamMatcher::Prefix(v), - sdk::types::StreamMatcher::None => StreamMatcher::Prefix(Default::default()), + sdk::types::StreamMatcher::Exact(v) => Some(StreamMatcher::Exact(v)), + sdk::types::StreamMatcher::Prefix(v) => Some(StreamMatcher::Prefix(v)), + sdk::types::StreamMatcher::None => None, } } } -impl From for AccessTokenMatcher { - fn from(matcher: sdk::types::AccessTokenMatcher) -> Self { +impl AccessTokenMatcher { + /// Converts an SDK matcher, returning `None` for a matcher that matches no resources. + fn from_sdk(matcher: sdk::types::AccessTokenMatcher) -> Option { match matcher { - sdk::types::AccessTokenMatcher::Exact(v) => AccessTokenMatcher::Exact(v), - sdk::types::AccessTokenMatcher::Prefix(v) => AccessTokenMatcher::Prefix(v), - sdk::types::AccessTokenMatcher::None => AccessTokenMatcher::Prefix(Default::default()), + sdk::types::AccessTokenMatcher::Exact(v) => Some(AccessTokenMatcher::Exact(v)), + sdk::types::AccessTokenMatcher::Prefix(v) => Some(AccessTokenMatcher::Prefix(v)), + sdk::types::AccessTokenMatcher::None => None, } } } @@ -776,9 +763,9 @@ pub struct AccessTokenScope { impl From for AccessTokenScope { fn from(scope: sdk::types::AccessTokenScope) -> Self { AccessTokenScope { - basins: scope.basins.map(Into::into), - streams: scope.streams.map(Into::into), - access_tokens: scope.access_tokens.map(Into::into), + basins: scope.basins.and_then(BasinMatcher::from_sdk), + streams: scope.streams.and_then(StreamMatcher::from_sdk), + access_tokens: scope.access_tokens.and_then(AccessTokenMatcher::from_sdk), op_group_perms: scope.op_group_perms.map(Into::into), ops: scope.ops.into_iter().map(Operation::from).collect(), } @@ -946,6 +933,37 @@ mod tests { ); } + #[test] + fn match_none_scope_matchers_render_as_no_access() { + colored::control::set_override(false); + + // On the wire, an empty exact name means "match no resources". + let scope: s2_api::v1::access::AccessTokenScope = + serde_json::from_value(serde_json::json!({ + "basins": { "exact": "" }, + "streams": { "exact": "" }, + "access_tokens": { "exact": "" }, + })) + .unwrap(); + let scope = s2_sdk::types::AccessTokenScope::from(scope); + let info = AccessTokenInfo { + id: "tok".to_owned(), + expires_at: None, + auto_prefix_streams: false, + scope: scope.into(), + }; + + assert_eq!( + info.summary_block(3), + "tok expires never\n\ + \x20 basins=∅ streams=∅ tokens=∅ perms=none ops=0" + ); + let json = serde_json::to_value(&info.scope).unwrap(); + assert!(json["basins"].is_null(), "{json}"); + assert!(json["streams"].is_null(), "{json}"); + assert!(json["access_tokens"].is_null(), "{json}"); + } + #[test] fn op_group_perms_summary_omits_empty_groups() { assert_eq!( diff --git a/cli/tests/integration.rs b/cli/tests/integration.rs index b193551f..7d0152f0 100644 --- a/cli/tests/integration.rs +++ b/cli/tests/integration.rs @@ -563,6 +563,95 @@ fn append_from_stdin() { cleanup_stream(&basin, &stream); } +#[test] +#[serial] +fn append_with_stream_config() { + let basin = unique_name("test-cli-csoa-cfg"); + s2().args([ + "create-basin", + &basin, + "--retention-policy", + "7d", + "--create-stream-on-append", + ]) + .assert() + .success(); + wait_for_basin(&basin); + + let stream = unique_name("test-csoa-new"); + let uri = format!("s2://{basin}/{stream}"); + s2().args([ + "append", + &uri, + "--format", + "text", + "--input", + "-", + "--retention-policy", + "1h", + "--delete-on-empty-min-age", + "5m", + ]) + .write_stdin("first record\n") + .assert() + .success(); + + s2().args(["get-stream-config", &uri]) + .assert() + .success() + .stdout( + predicate::str::contains("1h") + .and(predicate::str::contains("5m")) + .and(predicate::str::contains("7days").not()), + ); + + s2().args([ + "append", + &uri, + "--format", + "text", + "--input", + "-", + "--retention-policy", + "2h", + ]) + .write_stdin("second record\n") + .assert() + .success(); + + s2().args(["get-stream-config", &uri]) + .assert() + .success() + .stdout(predicate::str::contains("1h").and(predicate::str::contains("2h").not())); + + cleanup_stream(&basin, &stream); + cleanup_basin(&basin); +} + +#[test] +#[serial] +fn read_with_stream_config() { + let basin = unique_name("test-cli-csor-cfg"); + s2().args(["create-basin", &basin, "--create-stream-on-read"]) + .assert() + .success(); + wait_for_basin(&basin); + + let stream = unique_name("test-csor-new"); + let uri = format!("s2://{basin}/{stream}"); + s2().args(["read", &uri, "--count", "1", "--retention-policy", "1h"]) + .assert() + .failure(); + + s2().args(["get-stream-config", &uri]) + .assert() + .success() + .stdout(predicate::str::contains("1h")); + + cleanup_stream(&basin, &stream); + cleanup_basin(&basin); +} + #[test] #[serial] fn tail_stream() { diff --git a/cli/tests/lite_wal.rs b/cli/tests/lite_wal.rs new file mode 100644 index 00000000..609ec24d --- /dev/null +++ b/cli/tests/lite_wal.rs @@ -0,0 +1,227 @@ +use std::{ + fs::{self, File}, + io::{Read, Write}, + net::{SocketAddr, TcpListener, TcpStream}, + path::Path, + process::{Child, Command as ProcessCommand, Stdio}, + time::{Duration, Instant}, +}; + +use assert_cmd::Command; +use tempfile::TempDir; + +fn base_command(home: &Path) -> ProcessCommand { + let mut cmd = ProcessCommand::new(assert_cmd::cargo::cargo_bin!("s2")); + cmd.env_clear() + .env("HOME", home) + .env("XDG_CONFIG_HOME", home.join(".config")) + .env("APPDATA", home) + .env("USERPROFILE", home) + .env("NO_COLOR", "1") + .env("RUST_LOG", "warn"); + cmd +} + +fn command(home: &Path) -> Command { + let mut cmd = Command::from_std(base_command(home)); + cmd.timeout(Duration::from_secs(30)); + cmd +} + +#[test] +fn wal_store_requires_a_persistent_main_store_and_one_backend() { + let home = tempfile::tempdir().unwrap(); + for args in [ + vec!["lite", "--wal-bucket", "wal"], + vec!["lite", "--wal-local-root", "wal"], + vec![ + "lite", + "--bucket", + "main", + "--wal-bucket", + "wal", + "--wal-local-root", + "wal", + ], + vec!["lite", "--bucket", "main", "--local-root", "main"], + ] { + command(home.path()).args(args).assert().failure().code(2); + } + command(home.path()) + .env("S2LITE_WAL_BUCKET", "wal") + .args(["lite"]) + .assert() + .failure() + .code(2); +} + +struct Server { + child: Child, + address: SocketAddr, +} + +impl Server { + fn start(root: &Path, attempt: &str) -> Self { + for bind_attempt in 0.. { + let log_path = root.join(format!("server-{attempt}-{bind_attempt}.log")); + match Self::spawn(root, &log_path) { + Ok(server) => return server, + Err(log) if bind_attempt < 5 && log.contains("Address already in use") => { + continue; + } + Err(log) => panic!("Server exited before becoming healthy: {log}"), + } + } + unreachable!() + } + + fn spawn(root: &Path, log_path: &Path) -> Result { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let address = listener.local_addr().unwrap(); + drop(listener); + let log = File::create(log_path).unwrap(); + let mut cmd = base_command(root); + cmd.args([ + "lite", + "--path", + "db", + "--port", + &address.port().to_string(), + "--local-root", + ]) + .arg(root.join("main")) + .env("S2LITE_WAL_LOCAL_ROOT", root.join("wal")) + .env("SL8_FLUSH_INTERVAL", "1ms") + .env("SL8_L0_SST_SIZE_BYTES", "65536"); + let child = cmd + .stdin(Stdio::null()) + .stdout(log.try_clone().unwrap()) + .stderr(log) + .spawn() + .unwrap(); + let mut server = Self { child, address }; + let deadline = Instant::now() + Duration::from_secs(30); + loop { + if server.child.try_wait().unwrap().is_some() { + return Err(fs::read_to_string(log_path).unwrap()); + } + if let Ok(mut stream) = TcpStream::connect_timeout(&address, Duration::from_millis(100)) + { + stream + .set_read_timeout(Some(Duration::from_secs(1))) + .unwrap(); + stream + .set_write_timeout(Some(Duration::from_secs(1))) + .unwrap(); + let mut response = [0; 256]; + if stream + .write_all( + b"GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + ) + .is_ok() + && let Ok(n) = stream.read(&mut response) + && response[..n].starts_with(b"HTTP/1.1 200") + { + return Ok(server); + } + } + assert!( + Instant::now() < deadline, + "Server did not become healthy: {}", + fs::read_to_string(log_path).unwrap() + ); + std::thread::sleep(Duration::from_millis(25)); + } + } + + fn client(&self, root: &Path, args: &[&str], input: Option<&str>) -> String { + let endpoint = format!("http://{}", self.address); + let mut cmd = command(root); + cmd.env("S2_ACCOUNT_ENDPOINT", &endpoint) + .env("S2_BASIN_ENDPOINT", &endpoint) + .env("S2_ACCESS_TOKEN", "test") + .args(args); + if let Some(input) = input { + cmd.write_stdin(input); + } + let output = cmd.output().unwrap(); + assert!( + output.status.success(), + "{args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() + } + + fn crash(&mut self) { + self.child.kill().unwrap(); + self.child.wait().unwrap(); + } +} + +impl Drop for Server { + fn drop(&mut self) { + let _ = self.child.kill(); + let _ = self.child.wait(); + } +} + +fn has_sst(path: &Path) -> bool { + fs::read_dir(path).is_ok_and(|entries| { + entries + .flatten() + .any(|entry| entry.path().extension().is_some_and(|ext| ext == "sst")) + }) +} + +#[test] +fn separate_wal_routes_files_and_recovers_acknowledged_records_after_restart() { + let root: TempDir = tempfile::tempdir().unwrap(); + let mut server = Server::start(root.path(), "before"); + server.client(root.path(), &["create-basin", "wal-test"], None); + server.client( + root.path(), + &["create-stream", "s2://wal-test/recovery"], + None, + ); + let mut input = (0..64) + .map(|i| format!("record-{i}-{}\n", "x".repeat(4096))) + .collect::(); + server.client( + root.path(), + &["append", "s2://wal-test/recovery"], + Some(&input), + ); + let deadline = Instant::now() + Duration::from_secs(10); + while !has_sst(&root.path().join("main/db/compacted")) { + assert!(Instant::now() < deadline, "Main-store SST was not flushed"); + std::thread::sleep(Duration::from_millis(25)); + } + // Leave a final small write in the WAL after the earlier data reaches L0. + let last = "last-acknowledged-record\n"; + server.client( + root.path(), + &["append", "s2://wal-test/recovery"], + Some(last), + ); + input.push_str(last); + assert!(has_sst(&root.path().join("wal/db/wal"))); + assert!(!root.path().join("main/db/wal").exists()); + assert!(!root.path().join("wal/db/compacted").exists()); + assert!(!root.path().join("wal/db/manifest").exists()); + server.crash(); + let restarted = Server::start(root.path(), "after"); + let read = restarted.client( + root.path(), + &[ + "read", + "s2://wal-test/recovery", + "--seq-num", + "0", + "--count", + "65", + ], + None, + ); + assert_eq!(read, input); +} diff --git a/common/CHANGELOG.md b/common/CHANGELOG.md index 5d7db4c4..8fa9a24a 100644 --- a/common/CHANGELOG.md +++ b/common/CHANGELOG.md @@ -2,6 +2,30 @@ All notable changes to this project will be documented in this file. +## [0.42.0] - 2026-09-25 + +### Features + +- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) + + + +## [0.41.3] - 2026-09-22 + +### Miscellaneous Tasks + +- Update Cargo.toml dependencies + + + +## [0.41.2] - 2026-09-10 + +### Bug Fixes + +- Reject NUL bytes in stream names and access token IDs ([#728](https://github.com/s2-streamstore/s2/issues/728)) + + + ## [0.41.1] - 2026-07-22 ### Miscellaneous Tasks diff --git a/common/Cargo.toml b/common/Cargo.toml index 202963a7..acdd4354 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-common" -version = "0.41.1" +version = "0.42.0" description = "Common stuff for client and servers for S2, the durable streams API" edition.workspace = true license.workspace = true diff --git a/common/src/config.rs b/common/src/config.rs index 83526ec2..8fc17e45 100644 --- a/common/src/config.rs +++ b/common/src/config.rs @@ -2,8 +2,9 @@ //! //! Stream configuration uses three representations: //! -//! - Resolved (`StreamConfig`, `TimestampingConfig`, `DeleteOnEmptyConfig`): concrete values, -//! produced by merging optional configs with defaults using `merge()`. +//! - Merged (`StreamConfig`, `TimestampingConfig`, `DeleteOnEmptyConfig`): values produced by +//! merging optional configs with defaults using `merge()`. Storage class remains unspecified when +//! neither the stream nor basin supplies one. //! //! - Optional (`OptionalStreamConfig`, `OptionalTimestampingConfig`, //! `OptionalDeleteOnEmptyConfig`): partial configuration layers, where `None` means "not set at @@ -17,39 +18,17 @@ //! applies the inner reconfiguration to the existing value, while `Specified(None)` //! clears it to the default. //! -//! `merge()` resolves optional configs into resolved configs with precedence: -//! stream-level → basin-level → system default (via `Option::or` chaining). +//! `merge()` applies configuration layers with precedence: +//! stream-level → basin-level → field default. //! //! Basin config also carries basin-level knobs like `stream_cipher`, //! `create_stream_on_append`, and `create_stream_on_read`. use std::time::Duration; -use crate::{ValidationError, encryption::EncryptionAlgorithm, maybe::Maybe}; +use compact_str::CompactString; -#[derive( - Debug, - Default, - Clone, - Copy, - strum::Display, - strum::IntoStaticStr, - strum::EnumIter, - strum::FromRepr, - strum::EnumString, - PartialEq, - Eq, - Hash, -)] -#[cfg_attr(feature = "clap", derive(clap::ValueEnum))] -#[repr(u8)] -pub enum StorageClass { - #[strum(serialize = "standard")] - Standard = 1, - #[default] - #[strum(serialize = "express")] - Express = 2, -} +use crate::{ValidationError, encryption::EncryptionAlgorithm, maybe::Maybe}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RetentionPolicy { @@ -110,7 +89,7 @@ impl DeleteOnEmptyConfig { #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct StreamConfig { - pub storage_class: StorageClass, + pub storage_class: Option, pub retention_policy: RetentionPolicy, pub timestamping: TimestampingConfig, pub delete_on_empty: DeleteOnEmptyConfig, @@ -129,7 +108,7 @@ pub struct DeleteOnEmptyReconfiguration { #[derive(Debug, Clone, Default)] pub struct StreamReconfiguration { - pub storage_class: Maybe>, + pub storage_class: Maybe>, pub retention_policy: Maybe>, pub timestamping: Maybe>, pub delete_on_empty: Maybe>, @@ -217,7 +196,7 @@ impl From for OptionalDeleteOnEmptyConfig { #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct OptionalStreamConfig { - pub storage_class: Option, + pub storage_class: Option, pub retention_policy: Option, pub timestamping: OptionalTimestampingConfig, pub delete_on_empty: OptionalDeleteOnEmptyConfig, @@ -258,10 +237,7 @@ impl OptionalStreamConfig { } pub fn merge(self, basin_defaults: Self) -> StreamConfig { - let storage_class = self - .storage_class - .or(basin_defaults.storage_class) - .unwrap_or_default(); + let storage_class = self.storage_class.or(basin_defaults.storage_class); let retention_policy = self .retention_policy @@ -291,7 +267,7 @@ impl From for StreamConfig { } = value; Self { - storage_class: storage_class.unwrap_or_default(), + storage_class, retention_policy: retention_policy.unwrap_or_default(), timestamping: timestamping.into(), delete_on_empty: delete_on_empty.into(), @@ -309,7 +285,7 @@ impl From for OptionalStreamConfig { } = value; Self { - storage_class: Some(storage_class), + storage_class, retention_policy: Some(retention_policy), timestamping: timestamping.into(), delete_on_empty: delete_on_empty.into(), diff --git a/common/src/location.rs b/common/src/location.rs index 6ce67076..86605447 100644 --- a/common/src/location.rs +++ b/common/src/location.rs @@ -146,6 +146,8 @@ impl From for CompactString { pub struct LocationInfo { pub name: LocationName, pub is_private: bool, + pub storage_classes: Vec, + pub default_storage_class: CompactString, } #[cfg(test)] diff --git a/deny.toml b/deny.toml index 48b28369..24aac6b4 100644 --- a/deny.toml +++ b/deny.toml @@ -6,8 +6,6 @@ ignore = [ # Transitive deps via slatedb/foyer — not actionable. "RUSTSEC-2024-0436", # paste unmaintained "RUSTSEC-2025-0141", # bincode unmaintained - # Transitive dep via tabled_derive — no fixed release available. - "RUSTSEC-2026-0173", # proc-macro-error2 unmaintained # Transitive dep via object_store (slatedb) — fix requires object_store >=0.13. "RUSTSEC-2026-0194", # quick-xml quadratic attribute check "RUSTSEC-2026-0195", # quick-xml unbounded namespace allocation diff --git a/lite/CHANGELOG.md b/lite/CHANGELOG.md index 3d7ff266..4f80ead3 100644 --- a/lite/CHANGELOG.md +++ b/lite/CHANGELOG.md @@ -2,6 +2,81 @@ All notable changes to this project will be documented in this file. +## [0.43.0] - 2026-09-25 + +### Features + +- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) + +### Bug Fixes + +- Gate basin deletion draining on per-basin progress ([#779](https://github.com/s2-streamstore/s2/issues/779)) + + + +## [0.42.14] - 2026-09-24 + +### Bug Fixes + +- Coalesce delete-on-empty scheduling ([#772](https://github.com/s2-streamstore/s2/issues/772)) + + + +## [0.42.13] - 2026-09-22 + +### Features + +- Support a separate WAL object store ([#766](https://github.com/s2-streamstore/s2/issues/766)) + +### Bug Fixes + +- Keep streamers alive until pending writes settle ([#757](https://github.com/s2-streamstore/s2/issues/757)) +- Await durable deletion markers on retries ([#756](https://github.com/s2-streamstore/s2/issues/756)) +- Prevent stale or missed stream config updates ([#759](https://github.com/s2-streamstore/s2/issues/759)) +- Apply the at-tail read guard after resolving a timestamp start ([#762](https://github.com/s2-streamstore/s2/issues/762)) +- Prevent deletion races when recreating streams ([#760](https://github.com/s2-streamstore/s2/issues/760)) +- Preserve SSE read budgets across reconnects ([#727](https://github.com/s2-streamstore/s2/issues/727)) + +### Performance + +- Avoid copying serialized append buffers ([#763](https://github.com/s2-streamstore/s2/issues/763)) +- Reuse acknowledgement queue capacity ([#764](https://github.com/s2-streamstore/s2/issues/764)) + + + +## [0.42.12] - 2026-09-16 + +### Miscellaneous Tasks + +- Update Cargo.lock dependencies + + + +## [0.42.11] - 2026-09-11 + +### Features + +- `s2-stream-config` header for auto-created streams ([#718](https://github.com/s2-streamstore/s2/issues/718)) + + + +## [0.42.10] - 2026-09-11 + +### Miscellaneous Tasks + +- Update Cargo.lock dependencies + + + +## [0.42.9] - 2026-09-10 + +### Bug Fixes + +- Close SlateDB on graceful shutdown +- Reject NUL bytes in stream names and access token IDs ([#728](https://github.com/s2-streamstore/s2/issues/728)) + + + ## [0.42.8] - 2026-09-01 ### Miscellaneous Tasks diff --git a/lite/Cargo.toml b/lite/Cargo.toml index bb5c7709..8f0d8988 100644 --- a/lite/Cargo.toml +++ b/lite/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-lite" -version = "0.42.8" +version = "0.43.0" description = "Lightweight server implementation of S2, the durable streams API, backed by object storage" edition.workspace = true license.workspace = true diff --git a/lite/src/backend/append.rs b/lite/src/backend/append.rs index 82343edb..aa83f752 100644 --- a/lite/src/backend/append.rs +++ b/lite/src/backend/append.rs @@ -7,6 +7,7 @@ use std::{ use futures::{Stream, StreamExt as _, future::OptionFuture, stream::FuturesOrdered}; use s2_common::{ basin::BasinName, + config::OptionalStreamConfig, encryption::{EncryptionKey, EncryptionSpec}, record::{SeqNum, StreamPosition}, stream::{AppendAck, AppendInput, StreamName}, @@ -14,20 +15,26 @@ use s2_common::{ use s2_storage::record::encrypt_append_input; use tokio::sync::oneshot; -use super::{Backend, StreamHandle}; +use super::{Backend, StreamHandle, core::AutoCreateOn}; use crate::backend::error::{AppendError, AppendErrorInternal, StorageError}; impl Backend { + /// Open a stream for an append or append session. + /// + /// `stream_config` is applied if the stream is created on append. Unset fields inherit the + /// basin's default stream configuration. Ignored if the stream already exists. pub async fn open_for_append( &self, basin: &BasinName, stream: &StreamName, encryption_key: Option, + stream_config: OptionalStreamConfig, ) -> Result { self.stream_handle_with_auto_create::( basin, stream, - |config| config.create_stream_on_append, + AutoCreateOn::Append, + stream_config, |cipher| Ok(EncryptionSpec::resolve(cipher, encryption_key)?), ) .await @@ -134,6 +141,8 @@ pub fn admit( } } +const MIN_PENDING_APPENDS_CAPACITY: usize = 16; + #[derive(Debug, Default)] pub struct PendingAppends { queue: VecDeque, @@ -188,8 +197,12 @@ impl PendingAppends { } // Lots of small appends could cause this, // as we bound only on total bytes not num batches. - if self.queue.capacity() >= 4 * self.queue.len() { - self.queue.shrink_to(self.queue.len() * 2); + // Keep a small buffer to reuse across ordinary drain/refill cycles. + if self.queue.capacity() > MIN_PENDING_APPENDS_CAPACITY + && self.queue.capacity() >= 4 * self.queue.len() + { + self.queue + .shrink_to((self.queue.len() * 2).max(MIN_PENDING_APPENDS_CAPACITY)); } } @@ -271,3 +284,115 @@ impl BlockedReplySender { let _ = self.tx.send(reply); } } + +#[cfg(test)] +mod tests { + use super::*; + + fn position(seq_num: SeqNum) -> StreamPosition { + StreamPosition { + seq_num, + timestamp: seq_num, + } + } + + fn accept( + pending: &mut PendingAppends, + seq_num: SeqNum, + ) -> oneshot::Receiver> { + let (tx, rx) = oneshot::channel(); + let ticket = admit(tx, None).expect("open receiver"); + pending.accept(ticket, position(seq_num)..position(seq_num + 1)); + rx + } + + #[rstest::rstest] + #[case(1)] + #[case(16)] + fn small_append_bursts_reuse_queue_capacity(#[case] burst_size: SeqNum) { + let mut pending = PendingAppends::new(); + let mut retained_capacity = None; + for burst in 0..4 { + let start = burst * burst_size; + let receivers: Vec<_> = (start..start + burst_size) + .map(|seq_num| accept(&mut pending, seq_num)) + .collect(); + let capacity = *retained_capacity.get_or_insert(pending.queue.capacity()); + assert_eq!(pending.queue.capacity(), capacity); + + // The streamer advances durability one append at a time, even when + // the whole burst became durable in the same WAL flush. + for (offset, mut rx) in receivers.into_iter().enumerate() { + let seq_num = start + offset as SeqNum; + assert!(matches!( + rx.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + )); + pending.on_stable(position(seq_num + 1)); + let ack = rx + .try_recv() + .expect("ack delivered") + .expect("append accepted"); + assert_eq!(ack.start, position(seq_num)); + assert_eq!(ack.end, position(seq_num + 1)); + assert_eq!(ack.tail, position(seq_num + 1)); + assert_eq!(pending.queue.capacity(), capacity); + } + assert!(pending.queue.is_empty()); + } + } + + #[test] + fn unused_queue_stays_unallocated() { + let mut pending = PendingAppends::new(); + pending.on_stable(StreamPosition::MIN); + assert_eq!(pending.queue.capacity(), 0); + } + + #[test] + fn large_append_burst_releases_excess_capacity() { + let mut pending = PendingAppends::new(); + let mut receivers: Vec<_> = (0..128) + .map(|seq_num| accept(&mut pending, seq_num)) + .collect(); + let peak_capacity = pending.queue.capacity(); + + pending.on_stable(position(96)); + assert_eq!(pending.queue.len(), 32); + let reduced_capacity = pending.queue.capacity(); + assert!(reduced_capacity < peak_capacity); + for (seq_num, rx) in receivers.iter_mut().enumerate() { + if seq_num < 96 { + let ack = rx + .try_recv() + .expect("ack delivered") + .expect("append accepted"); + assert_eq!(ack.start, position(seq_num as SeqNum)); + assert_eq!(ack.end, position(seq_num as SeqNum + 1)); + assert_eq!(ack.tail, position(96)); + } else { + assert!(matches!( + rx.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + )); + } + } + + // A small change in occupancy should not immediately shrink again. + pending.on_stable(position(97)); + assert_eq!(pending.queue.capacity(), reduced_capacity); + pending.on_stable(position(128)); + assert!(pending.queue.is_empty()); + assert!(pending.queue.capacity() > 0); + assert!(pending.queue.capacity() <= MIN_PENDING_APPENDS_CAPACITY); + for (seq_num, rx) in receivers.iter_mut().enumerate().skip(96) { + let ack = rx + .try_recv() + .expect("ack delivered") + .expect("append accepted"); + assert_eq!(ack.start, position(seq_num as SeqNum)); + assert_eq!(ack.end, position(seq_num as SeqNum + 1)); + assert_eq!(ack.tail, position(if seq_num == 96 { 97 } else { 128 })); + } + } +} diff --git a/lite/src/backend/basins.rs b/lite/src/backend/basins.rs index c37511af..73f1cb01 100644 --- a/lite/src/backend/basins.rs +++ b/lite/src/backend/basins.rs @@ -11,7 +11,11 @@ use slatedb::{ }; use time::OffsetDateTime; -use super::{Backend, bgtasks::BgtaskTrigger, store::db_txn_get}; +use super::{ + Backend, + bgtasks::BgtaskTrigger, + store::{db_txn_commit_durable, db_txn_get, db_txn_get_with}, +}; use crate::backend::{ error::{ BasinAlreadyExistsError, BasinDeletionPendingError, BasinNotFoundError, DeleteBasinError, @@ -63,6 +67,8 @@ impl Backend { Ok(Page::new(basins, has_more)) } + /// Any outcome asserting the basin exists — `Created`, `Updated`, `Noop`, + /// or `BasinAlreadyExists` — is readable at `DurabilityLevel::Remote`. pub async fn provision_basin( &self, basin: BasinName, @@ -73,7 +79,12 @@ impl Backend { let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; - let existing_meta = db_txn_get(&txn, &meta_key, kv::basin_meta::deser_value).await?; + // A transaction can see metadata that has not been flushed yet. + let (existing_meta, existing_seq) = db_txn_get_with(&txn, &meta_key, |entry| { + Ok((kv::basin_meta::deser_value(entry.value)?, entry.seq)) + }) + .await? + .unzip(); if let Some(existing_meta) = &existing_meta && existing_meta.deleted_at.is_some() { @@ -85,7 +96,7 @@ impl Backend { let new_creation_idempotency_key = request_token .as_ref() .map(|req_token| creation_idempotency_key(req_token, &config)); - return if new_creation_idempotency_key.is_some() + let result = if new_creation_idempotency_key.is_some() && existing.creation_idempotency_key == new_creation_idempotency_key { Ok(ProvisionResult::Noop(BasinInfo { @@ -97,6 +108,10 @@ impl Backend { } else { Err(BasinAlreadyExistsError { basin }.into()) }; + drop(txn); + self.await_durable_seq(existing_seq.expect("existing meta was read")) + .await?; + return result; } (Some(existing), ProvisionMode::Ensure) => { let meta = kv::basin_meta::BasinMeta { @@ -130,11 +145,15 @@ impl Backend { }), }; - if !matches!(&outcome, ProvisionResult::Noop(_)) { + if matches!(&outcome, ProvisionResult::Noop(_)) { + drop(txn); + self.await_durable_seq(existing_seq.expect("noop implies existing meta")) + .await?; + } else { let meta = outcome.inner(); txn.put(&meta_key, kv::basin_meta::ser_value(meta))?; - txn.commit().await?; + db_txn_commit_durable(txn).await?; } Ok(outcome.map(|meta| BasinInfo { @@ -179,7 +198,7 @@ impl Backend { txn.put(&meta_key, kv::basin_meta::ser_value(&meta))?; - txn.commit().await?; + db_txn_commit_durable(txn).await?; Ok(meta.config) } @@ -187,7 +206,11 @@ impl Backend { pub async fn delete_basin(&self, basin: BasinName) -> Result<(), DeleteBasinError> { let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; let meta_key = kv::basin_meta::ser_key(&basin); - let Some(mut meta) = db_txn_get(&txn, &meta_key, kv::basin_meta::deser_value).await? else { + let Some((mut meta, seq)) = db_txn_get_with(&txn, &meta_key, |entry| { + Ok((kv::basin_meta::deser_value(entry.value)?, entry.seq)) + }) + .await? + else { return Err(BasinNotFoundError { basin }.into()); }; if meta.deleted_at.is_none() { @@ -197,9 +220,13 @@ impl Backend { kv::basin_deletion_pending::ser_key(&basin), kv::basin_deletion_pending::ser_value(&StreamNameStartAfter::default()), )?; - txn.commit().await?; - self.bgtask_trigger(BgtaskTrigger::BasinDeletion); + db_txn_commit_durable(txn).await?; + } else { + // A retry may observe the marker before the first delete has flushed. + drop(txn); + self.await_durable_seq(seq).await?; } + self.bgtask_trigger(BgtaskTrigger::BasinDeletion); Ok(()) } } diff --git a/lite/src/backend/bgtasks/basin_deletion.rs b/lite/src/backend/bgtasks/basin_deletion.rs index e11563be..1dbb4344 100644 --- a/lite/src/backend/bgtasks/basin_deletion.rs +++ b/lite/src/backend/bgtasks/basin_deletion.rs @@ -10,6 +10,7 @@ use slatedb::{ }; use tracing::instrument; +use super::{ItemProgress, PageProgress}; use crate::backend::{ Backend, error::{BasinDeletionError, ListStreamsError, StorageError}, @@ -25,17 +26,17 @@ impl Backend { if page.values.is_empty() { return Ok(page.has_more); } + let mut progress = PageProgress::new(page.has_more); let mut processed = stream::iter(page.values) .map(|(basin, cursor)| { let backend = self.clone(); async move { backend.process_basin_deletion(basin, cursor).await } }) .buffer_unordered(CONCURRENCY); - let mut has_more = page.has_more; while let Some(result) = processed.next().await { - has_more |= result?; + progress.record(result?); } - Ok(has_more) + Ok(progress.should_continue()) } async fn list_basin_deletion_pending( @@ -68,7 +69,7 @@ impl Backend { &self, basin: BasinName, cursor: StreamNameStartAfter, - ) -> Result { + ) -> Result { let request = ListStreamsRequest { prefix: StreamNamePrefix::default(), start_after: cursor.clone(), @@ -94,20 +95,21 @@ impl Backend { if page.has_more { self.set_basin_deletion_cursor(&basin, &last_stream.expect("non-empty stream page")) .await?; - Ok(true) + Ok(ItemProgress::Advanced) } else if last_stream.is_some() || !cursor.as_ref().is_empty() { // Streams still pending deletion or cursor was advanced past // earlier entries. Reset cursor so the next tick re-scans from - // the beginning. + // the beginning. A reset is not progress: counting it would + // rescan a multi-page basin in a tight loop. if !cursor.as_ref().is_empty() { self.set_basin_deletion_cursor(&basin, &StreamNameStartAfter::default()) .await?; } - Ok(false) + Ok(ItemProgress::Blocked) } else { // No streams from the very beginning — safe to complete. self.complete_basin_deletion(&basin).await?; - Ok(false) + Ok(ItemProgress::Completed) } } @@ -122,7 +124,7 @@ impl Backend { kv::basin_deletion_pending::ser_key(basin), kv::basin_deletion_pending::ser_value(cursor), ); - self.db.write(batch).await?; + self.db.write(batch).await?.await_durable().await?; Ok(()) } @@ -131,7 +133,7 @@ impl Backend { let mut batch = WriteBatch::new(); batch.delete(kv::basin_meta::ser_key(basin)); batch.delete(kv::basin_deletion_pending::ser_key(basin)); - self.db.write(batch).await?; + self.db.write(batch).await?.await_durable().await?; Ok(()) } } @@ -149,7 +151,7 @@ mod tests { use time::OffsetDateTime; use super::super::tests::test_backend; - use crate::backend::{Backend, kv}; + use crate::backend::{Backend, kv, test_util::DbWriteTestExt as _}; fn basin_meta(deleted_at: Option) -> kv::basin_meta::BasinMeta { kv::basin_meta::BasinMeta { @@ -185,16 +187,16 @@ mod tests { kv::basin_meta::ser_key(basin), kv::basin_meta::ser_value(&basin_meta(Some(OffsetDateTime::now_utc()))), ) - .await - .unwrap(); + .assert_durable() + .await; backend .db .put( kv::basin_deletion_pending::ser_key(basin), kv::basin_deletion_pending::ser_value(&StreamNameStartAfter::default()), ) - .await - .unwrap(); + .assert_durable() + .await; } async fn seed_tombstoned_streams(backend: &Backend, basin: &BasinName, count: usize) { @@ -207,7 +209,7 @@ mod tests { kv::stream_meta::ser_value(&stream_meta(Some(deleted_at))), ); } - backend.db.write(batch).await.unwrap(); + backend.db.write(batch).assert_durable().await; } #[tokio::test] @@ -221,16 +223,16 @@ mod tests { kv::basin_meta::ser_key(&basin), kv::basin_meta::ser_value(&basin_meta(Some(OffsetDateTime::now_utc()))), ) - .await - .unwrap(); + .assert_durable() + .await; backend .db .put( kv::basin_deletion_pending::ser_key(&basin), kv::basin_deletion_pending::ser_value(&StreamNameStartAfter::default()), ) - .await - .unwrap(); + .assert_durable() + .await; let has_more = backend.clone().tick_basin_deletion().await.unwrap(); assert!(!has_more); @@ -266,8 +268,17 @@ mod tests { kv::stream_meta::ser_key(&basin, &stream), kv::stream_meta::ser_value(&stream_meta(None)), ) - .await - .unwrap(); + .assert_durable() + .await; + + backend + .db + .put( + kv::stream_id_mapping::ser_key(crate::stream_id::StreamId::new(&basin, &stream)), + kv::stream_id_mapping::ser_value(&basin, &stream), + ) + .assert_durable() + .await; let has_more = backend.clone().tick_basin_deletion().await.unwrap(); assert!(!has_more); @@ -384,16 +395,16 @@ mod tests { kv::basin_meta::ser_key(&basin), kv::basin_meta::ser_value(&basin_meta(Some(OffsetDateTime::now_utc()))), ) - .await - .unwrap(); + .assert_durable() + .await; backend .db .put( kv::basin_deletion_pending::ser_key(&basin), kv::basin_deletion_pending::ser_value(&cursor), ) - .await - .unwrap(); + .assert_durable() + .await; // First tick resets cursor from past-end back to the beginning. let has_more = backend.clone().tick_basin_deletion().await.unwrap(); @@ -435,8 +446,8 @@ mod tests { kv::stream_meta::ser_key(&basin, &stream), kv::stream_meta::ser_value(&stream_meta(Some(deleted_at))), ) - .await - .unwrap(); + .assert_durable() + .await; let has_more = backend.clone().tick_basin_deletion().await.unwrap(); assert!(!has_more); @@ -474,8 +485,8 @@ mod tests { kv::stream_meta::ser_key(&basin, &stream), kv::stream_meta::ser_value(&stream_meta(Some(deleted_at))), ) - .await - .unwrap(); + .assert_durable() + .await; // First tick: blocked by tombstoned stream. let has_more = backend.clone().tick_basin_deletion().await.unwrap(); @@ -493,8 +504,8 @@ mod tests { backend .db .delete(kv::stream_meta::ser_key(&basin, &stream)) - .await - .unwrap(); + .assert_durable() + .await; // Second tick: no streams remain, completes. let has_more = backend.clone().tick_basin_deletion().await.unwrap(); diff --git a/lite/src/backend/bgtasks/mod.rs b/lite/src/backend/bgtasks/mod.rs index a92916f8..ed219387 100644 --- a/lite/src/backend/bgtasks/mod.rs +++ b/lite/src/backend/bgtasks/mod.rs @@ -9,6 +9,63 @@ mod basin_deletion; mod stream_doe; mod stream_trim; +/// Keep draining the backlog while at least one item makes progress. A page where +/// every item is blocked waits for the next tick instead of retrying in a tight loop. +struct PageProgress { + has_more: bool, + any_progressed: bool, +} + +enum ItemProgress { + /// Made progress, and more work for this item can run immediately. + Advanced, + Completed, + /// Made no progress; the item waits for a later tick. + Blocked, +} + +impl PageProgress { + fn new(has_more: bool) -> Self { + Self { + has_more, + any_progressed: false, + } + } + + fn record(&mut self, item: ItemProgress) { + match item { + ItemProgress::Advanced => { + self.has_more = true; + self.any_progressed = true; + } + ItemProgress::Completed => self.any_progressed = true, + ItemProgress::Blocked => {} + } + } + + fn record_result( + &mut self, + result: Result, + is_conflict: fn(&E) -> bool, + ) -> Result, E> { + match result { + Ok(value) => { + self.record(ItemProgress::Completed); + Ok(Some(value)) + } + Err(err) if is_conflict(&err) => { + self.record(ItemProgress::Blocked); + Ok(None) + } + Err(err) => Err(err), + } + } + + fn should_continue(&self) -> bool { + self.has_more && self.any_progressed + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum BgtaskTrigger { BasinDeletion, @@ -172,4 +229,13 @@ mod tests { assert_eq!(calls.load(Ordering::SeqCst), 3); } + + #[test] + fn blocked_page_waits_for_next_tick() { + let mut progress = PageProgress::new(true); + progress.record(ItemProgress::Blocked); + assert!(!progress.should_continue()); + progress.record(ItemProgress::Completed); + assert!(progress.should_continue()); + } } diff --git a/lite/src/backend/bgtasks/stream_doe.rs b/lite/src/backend/bgtasks/stream_doe.rs index d09b108d..d5c3a19d 100644 --- a/lite/src/backend/bgtasks/stream_doe.rs +++ b/lite/src/backend/bgtasks/stream_doe.rs @@ -1,21 +1,21 @@ -use std::time::Duration; - +use bytes::Bytes; use futures::{StreamExt, stream}; use indexmap::IndexMap; -use itertools::Itertools; -use s2_common::resources::Page; +use s2_common::{basin::BasinName, resources::Page, stream::StreamName}; use slatedb::{ - WriteBatch, + IsolationLevel, WriteBatch, config::{DurabilityLevel, ScanOptions}, }; -use tracing::instrument; +use super::PageProgress; use crate::{ backend::{ - Backend, + Backend, doe, error::{DeleteStreamError, StorageError, StreamDeleteOnEmptyError}, - kv::{self, timestamp::TimestampSecs}, - streamer::{TerminalTrimCondition, doe_arm_delay}, + kv, + store::{db_snapshot_get_with, db_txn_commit_durable, db_txn_get, db_txn_get_with}, + streamer::{TerminalTrimCondition, TerminalTrimOutcome}, + timestamp::TimestampSecs, }, stream_id::StreamId, }; @@ -23,780 +23,1222 @@ use crate::{ const PENDING_LIST_LIMIT: usize = 10_000; const CONCURRENCY: usize = 4; -#[derive(Debug)] -struct PendingDoeBatch { - entries: Vec, - last_write_cutoff: Option, +#[derive(Clone, Copy, Debug)] +struct PendingCheck { + stream_id: StreamId, + check: kv::stream_doe_state::Check, } -impl PendingDoeBatch { - fn new(entries: Vec) -> Self { - let last_write_cutoff = entries - .iter() - .filter_map(|entry| entry.last_write_cutoff()) - .max(); - Self { - entries, - last_write_cutoff, - } - } - - fn entries(&self) -> &[kv::stream_doe_deadline::Entry] { - &self.entries - } +struct CheckSnapshot { + revision: u64, + creation_seq: u64, + config_seq: u64, + basin: BasinName, + stream: StreamName, } impl Backend { pub(super) async fn tick_stream_doe(self) -> Result { - let now = TimestampSecs::now(); - let page = self.list_pending_stream_doe(now).await?; - if page.values.is_empty() { - return Ok(page.has_more); - } + // Drain legacy keys regardless of their old deadlines. Their timestamps + // and values have no role in the new scheduler or deletion eligibility. + let mut progress = self.migrate_stream_doe().await?; + let page = self.list_pending_stream_doe(TimestampSecs::now()).await?; + progress.has_more |= page.has_more; let mut processed = stream::iter(page.values) - .map(|(stream_id, pending)| { + .map(|pending| { let backend = self.clone(); - async move { backend.process_stream_doe(stream_id, pending).await } + async move { backend.process_stream_doe(pending).await } }) .buffer_unordered(CONCURRENCY); while let Some(result) = processed.next().await { - result?; + progress.record_result(result, StreamDeleteOnEmptyError::is_transaction_conflict)?; } - Ok(page.has_more) + Ok(progress.should_continue()) } async fn list_pending_stream_doe( &self, now: TimestampSecs, - ) -> Result, StorageError> { + ) -> Result, StorageError> { let scan_opts = ScanOptions { durability_filter: DurabilityLevel::Remote, ..Default::default() }; let mut it = self .db - .scan_with_options(kv::stream_doe_deadline::expired_key_range(now), &scan_opts) + .scan_with_options(kv::stream_doe_check::due_key_range(now), &scan_opts) .await?; - let mut pending: IndexMap> = IndexMap::new(); - let mut has_more = false; - let mut count = 0; + let mut pending = Vec::new(); while let Some(kv) = it.next().await? { - let (deadline, stream_id) = kv::stream_doe_deadline::deser_key(kv.key)?; - let min_age = kv::stream_doe_deadline::deser_value(kv.value)?; - assert!(deadline <= now); - pending - .entry(stream_id) - .or_default() - .push(kv::stream_doe_deadline::Entry { deadline, min_age }); - count += 1; - if count == PENDING_LIST_LIMIT { - has_more = true; - break; + let (stream_id, check) = kv::stream_doe_check::deser_key(kv.key)?; + pending.push(PendingCheck { stream_id, check }); + if pending.len() == PENDING_LIST_LIMIT { + return Ok(Page::new(pending, true)); } } - Ok(Page::new( - pending - .into_iter() - .map(|(stream_id, entries)| (stream_id, PendingDoeBatch::new(entries))) - .collect_vec(), - has_more, - )) + Ok(Page::new(pending, false)) } async fn process_stream_doe( &self, - stream_id: StreamId, - pending: PendingDoeBatch, + pending: PendingCheck, ) -> Result<(), StreamDeleteOnEmptyError> { - if let Some(last_write_cutoff) = pending.last_write_cutoff - && let Some((basin, stream)) = self.stream_id_mapping(stream_id).await? - { - match self - .delete_stream_with_condition( - basin, - stream, - TerminalTrimCondition::DeleteOnEmpty { last_write_cutoff }, - ) - .await - { - Ok(()) | Err(DeleteStreamError::StreamNotFound(_)) => {} - Err(err) => return Err(err.into()), - } - } - self.clear_doe_deadlines(stream_id, pending.entries()) - .await?; - Ok(()) - } - - #[instrument(ret, err, skip(self, pending), fields(num_deadlines = pending.len()))] - async fn clear_doe_deadlines( - &self, - stream_id: StreamId, - pending: &[kv::stream_doe_deadline::Entry], - ) -> Result<(), StorageError> { - let mut batch = WriteBatch::new(); - for entry in pending { - batch.delete(kv::stream_doe_deadline::ser_key(entry.deadline, stream_id)); - } - self.db.write(batch).await?; - Ok(()) - } - - pub(super) async fn arm_doe_on_full_trim( - &self, - stream_id: StreamId, - ) -> Result<(), StorageError> { - let Some((basin, stream)) = self.stream_id_mapping(stream_id).await? else { + let Some(snapshot) = self.observe_doe_check(pending).await? else { return Ok(()); }; - let Some(meta) = self - .db_get( - &kv::stream_meta::ser_key(&basin, &stream), - kv::stream_meta::deser_value, + let outcome = match self + .delete_stream_with_condition( + snapshot.basin.clone(), + snapshot.stream.clone(), + TerminalTrimCondition::DeleteOnEmpty { + expected_stream_creation_seq: snapshot.creation_seq, + expected_config_seq: snapshot.config_seq, + }, + ) + .await + { + Ok(outcome) => outcome, + Err(DeleteStreamError::StreamNotFound(_)) => TerminalTrimOutcome::Obsolete, + Err(err) => return Err(err.into()), + }; + self.finish_doe_check(pending, snapshot, outcome).await?; + Ok(()) + } + + /// Capture the scheduler revision before the streamer's asynchronous scan. + async fn observe_doe_check( + &self, + pending: PendingCheck, + ) -> Result, StorageError> { + // Observation needs a consistent view, but no transaction read set. The + // completion transaction validates this revision after the actor's scan. + let snapshot = self.db.snapshot().await?; + let state = db_snapshot_get_with( + &snapshot, + kv::stream_doe_state::ser_key(pending.stream_id), + |entry| Ok((kv::stream_doe_state::deser_value(entry.value)?, entry.seq)), + ) + .await?; + let Some((kv::stream_doe_state::State::Scheduled(_), revision)) = state + .filter(|(state, _)| *state == kv::stream_doe_state::State::Scheduled(pending.check)) + else { + drop(snapshot); + self.discard_doe_check(pending, state).await?; + return Ok(None); + }; + let mapping = db_snapshot_get_with( + &snapshot, + kv::stream_id_mapping::ser_key(pending.stream_id), + |entry| Ok((kv::stream_id_mapping::deser_value(entry.value)?, entry.seq)), + ) + .await?; + if let Some(((basin, stream), creation_seq)) = mapping + && revision >= creation_seq + && let Some((meta, config_seq)) = db_snapshot_get_with( + &snapshot, + kv::stream_meta::ser_key(&basin, &stream), + |entry| Ok((kv::stream_meta::deser_value(entry.value)?, entry.seq)), ) .await? - else { - return Ok(()); - }; - if meta.deleted_at.is_some() { + && meta.deleted_at.is_none() + && meta.config.delete_on_empty.min_age().is_some() + { + return Ok(Some(CheckSnapshot { + revision, + creation_seq, + config_seq, + basin, + stream, + })); + } + drop(snapshot); + self.discard_doe_check(pending, state).await?; + Ok(None) + } + + /// Obsolete work needs a transaction only when it is actually discarded. + /// Revalidate the snapshot so cleanup cannot erase a concurrent wake. + async fn discard_doe_check( + &self, + pending: PendingCheck, + observed: Option<(kv::stream_doe_state::State, u64)>, + ) -> Result<(), StorageError> { + let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; + let current = db_txn_get_with( + &txn, + kv::stream_doe_state::ser_key(pending.stream_id), + |entry| Ok((kv::stream_doe_state::deser_value(entry.value)?, entry.seq)), + ) + .await?; + if current != observed { return Ok(()); } - let Some(min_age) = meta.config.delete_on_empty.min_age() else { - return Ok(()); - }; - let deadline = TimestampSecs::after(doe_arm_delay(Duration::ZERO, min_age)); - self.db - .put( - kv::stream_doe_deadline::ser_key(deadline, stream_id), - kv::stream_doe_deadline::ser_value(min_age), - ) - .await?; + match current { + Some((kv::stream_doe_state::State::Scheduled(check), _)) if check == pending.check => { + doe::replace( + &txn, + pending.stream_id, + Some(kv::stream_doe_state::State::Scheduled(check)), + None, + )?; + } + _ => { + txn.delete(kv::stream_doe_check::ser_key( + pending.stream_id, + pending.check, + ))?; + } + } + db_txn_commit_durable(txn).await?; Ok(()) } -} -#[cfg(test)] -mod tests { - use std::{str::FromStr, time::Duration}; - - use s2_common::{ - basin::BasinName, - config::{ - BasinConfig, DeleteOnEmptyReconfiguration, OptionalStreamConfig, RetentionPolicy, - StreamReconfiguration, - }, - maybe::Maybe, - record::StreamPosition, - stream::StreamName, - }; - use slatedb::config::{DurabilityLevel, ScanOptions}; - use time::OffsetDateTime; - - use super::{super::tests::test_backend, PendingDoeBatch, TimestampSecs}; - use crate::{ - backend::{Backend, kv}, - stream_id::StreamId, - }; - - const MIN_AGE: Duration = Duration::from_secs(60); - - fn stream_meta_with_config( - config: OptionalStreamConfig, - created_at: OffsetDateTime, - ) -> kv::stream_meta::StreamMeta { - kv::stream_meta::StreamMeta { - config: config.into(), - cipher: None, - created_at, - deleted_at: None, - creation_idempotency_key: None, + async fn finish_doe_check( + &self, + pending: PendingCheck, + snapshot: CheckSnapshot, + outcome: TerminalTrimOutcome, + ) -> Result<(), StorageError> { + let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; + let state = db_txn_get_with( + &txn, + kv::stream_doe_state::ser_key(pending.stream_id), + |entry| Ok((kv::stream_doe_state::deser_value(entry.value)?, entry.seq)), + ) + .await?; + if state + != Some(( + kv::stream_doe_state::State::Scheduled(pending.check), + snapshot.revision, + )) + { + // A trim/configuration event owns the next check, even if it kept the + // same ticket. Consuming or postponing it here would lose that wake. + return Ok(()); } + // Successful deletion marks metadata and advances its sequence. Deferred + // outcomes must still belong to the configuration that was inspected. + if outcome != TerminalTrimOutcome::DeletionPending { + let config_seq = db_txn_get_with( + &txn, + kv::stream_meta::ser_key(&snapshot.basin, &snapshot.stream), + |entry| Ok(entry.seq), + ) + .await?; + if config_seq != Some(snapshot.config_seq) { + return Ok(()); + } + } + let next = match outcome { + TerminalTrimOutcome::RetryAt(at) => Some(kv::stream_doe_state::State::Scheduled( + kv::stream_doe_state::Check { + at, + id: rand::random(), + }, + )), + TerminalTrimOutcome::Parked => Some(kv::stream_doe_state::State::Parked), + TerminalTrimOutcome::DeletionPending | TerminalTrimOutcome::Obsolete => None, + }; + // Consuming the old check and installing its successor is one durable + // transaction, including when the caller disappears during commit. + doe::replace( + &txn, + pending.stream_id, + Some(kv::stream_doe_state::State::Scheduled(pending.check)), + next, + )?; + db_txn_commit_durable(txn).await?; + Ok(()) } - fn stream_meta_with_doe_min_age(min_age: Duration) -> kv::stream_meta::StreamMeta { - let mut config = OptionalStreamConfig::default(); - config.delete_on_empty.min_age = Some(min_age); - stream_meta_with_config(config, OffsetDateTime::now_utc()) - } - - async fn seed_stream_with_meta( - backend: &Backend, - basin: &BasinName, - stream: &StreamName, - meta: kv::stream_meta::StreamMeta, - ) -> StreamId { - let stream_id = StreamId::new(basin, stream); - backend - .db - .put( - kv::basin_meta::ser_key(basin), - kv::basin_meta::ser_value(&kv::basin_meta::BasinMeta { - config: BasinConfig::default(), - created_at: OffsetDateTime::now_utc(), - deleted_at: None, - creation_idempotency_key: None, - }), - ) - .await - .unwrap(); - backend - .db - .put( - kv::stream_meta::ser_key(basin, stream), - kv::stream_meta::ser_value(&meta), - ) - .await - .unwrap(); - backend - .db - .put( - kv::stream_id_mapping::ser_key(stream_id), - kv::stream_id_mapping::ser_value(basin, stream), - ) - .await - .unwrap(); - stream_id - } - - async fn list_doe_entries(backend: &Backend) -> Vec<(TimestampSecs, StreamId, Duration)> { + async fn migrate_stream_doe(&self) -> Result { let scan_opts = ScanOptions { durability_filter: DurabilityLevel::Remote, ..Default::default() }; - let mut it = backend + let mut it = self .db .scan_with_options( kv::key_type_range(kv::KeyType::StreamDeleteOnEmptyDeadline), &scan_opts, ) - .await - .unwrap(); - let mut entries = Vec::new(); - while let Some(kv) = it.next().await.unwrap() { - let (deadline, stream_id) = kv::stream_doe_deadline::deser_key(kv.key).unwrap(); - let min_age = kv::stream_doe_deadline::deser_value(kv.value).unwrap(); - entries.push((deadline, stream_id, min_age)); + .await?; + let mut pending: IndexMap> = IndexMap::new(); + let mut count = 0; + while let Some(entry) = it.next().await? { + let (_, stream_id, _) = kv::stream_doe_deadline::deser_key(entry.key.clone())?; + pending.entry(stream_id).or_default().push(entry.key); + count += 1; + if count == PENDING_LIST_LIMIT { + break; + } } - entries + let mut progress = PageProgress::new(count == PENDING_LIST_LIMIT); + if pending.is_empty() { + return Ok(progress); + } + let snapshot = self.db.snapshot().await?; + let mut migrations = stream::iter(pending) + .map(|(stream_id, keys)| self.migrate_doe_deadlines(&snapshot, stream_id, keys)) + .buffer_unordered(CONCURRENCY); + let mut cleanup = WriteBatch::new(); + while let Some(result) = migrations.next().await { + if let Some(keys) = + progress.record_result(result, StorageError::is_transaction_conflict)? + { + for key in keys { + cleanup.delete(key); + } + } + } + if !cleanup.is_empty() { + // Later enablement/recreation installs its own schedule. Only + // legacy keys are removed; new state and its revision stay intact. + // Durability also covers any commits observed by the snapshot. + self.db.write(cleanup).await?.await_durable().await?; + } + Ok(progress) } - async fn put_tail_position( - backend: &Backend, + async fn migrate_doe_deadlines( + &self, + snapshot: &slatedb::DbSnapshot, stream_id: StreamId, - position: StreamPosition, - ) -> TimestampSecs { - let key = kv::stream_tail_position::ser_key(stream_id); - backend - .db - .put(key.clone(), kv::stream_tail_position::ser_value(position)) - .await - .unwrap(); - let kv = backend - .db - .get_key_value(key) - .await - .unwrap() - .expect("tail position should exist"); - TimestampSecs::from_millis(kv.create_ts) - } - - fn deadline_after(write_timestamp: TimestampSecs, age: Duration) -> TimestampSecs { - let deadline_secs = u64::from(write_timestamp.as_u32()) - .saturating_add(age.as_secs()) - .min(u64::from(u32::MAX)) as u32; - TimestampSecs::from_secs(deadline_secs) - } - - async fn process_pending_stream_doe_at( - backend: &Backend, - stream_id: StreamId, - now: TimestampSecs, - ) { - let mut page = backend.list_pending_stream_doe(now).await.unwrap(); - assert!(!page.has_more); - assert_eq!(page.values.len(), 1); - let (pending_stream_id, pending) = page.values.pop().unwrap(); - assert_eq!(pending_stream_id, stream_id); - - backend - .process_stream_doe(stream_id, pending) - .await - .unwrap(); - } - - #[tokio::test] - async fn stream_doe_marks_deleted_and_clears_deadline() { - let backend = test_backend().await; - let basin = BasinName::from_str("doe-basin").unwrap(); - let stream = StreamName::from_str("doe-stream").unwrap(); - let stream_id = seed_stream_with_meta( - &backend, - &basin, - &stream, - stream_meta_with_doe_min_age(MIN_AGE), - ) - .await; - let write_timestamp = put_tail_position( - &backend, - stream_id, - StreamPosition { - seq_num: 1, - timestamp: 1234, - }, - ) - .await; - let deadline = deadline_after(write_timestamp, MIN_AGE); - - backend - .db - .put( - kv::stream_doe_deadline::ser_key(deadline, stream_id), - kv::stream_doe_deadline::ser_value(MIN_AGE), - ) - .await - .unwrap(); - - process_pending_stream_doe_at(&backend, stream_id, deadline).await; - - let meta = backend - .db - .get(kv::stream_meta::ser_key(&basin, &stream)) - .await - .unwrap() - .expect("stream meta should remain"); - let decoded = kv::stream_meta::deser_value(meta).unwrap(); - assert!(decoded.deleted_at.is_some()); - - let deadline_key = backend - .db - .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) - .await - .unwrap(); - assert!(deadline_key.is_none()); - } - - #[tokio::test] - async fn stream_doe_deletes_never_written_stream() { - let backend = test_backend().await; - let basin = BasinName::from_str("doe-basin-never").unwrap(); - let stream = StreamName::from_str("doe-stream-never").unwrap(); - let stream_id = StreamId::new(&basin, &stream); - let min_age = MIN_AGE; - let meta = stream_meta_with_doe_min_age(min_age); - - seed_stream_with_meta(&backend, &basin, &stream, meta).await; - - let write_timestamp = put_tail_position(&backend, stream_id, StreamPosition::MIN).await; - let deadline = deadline_after(write_timestamp, min_age); - backend - .db - .put( - kv::stream_doe_deadline::ser_key(deadline, stream_id), - kv::stream_doe_deadline::ser_value(min_age), - ) - .await - .unwrap(); - - process_pending_stream_doe_at(&backend, stream_id, deadline).await; - - let meta = backend - .db - .get(kv::stream_meta::ser_key(&basin, &stream)) - .await - .unwrap() - .expect("stream meta should remain"); - let decoded = kv::stream_meta::deser_value(meta).unwrap(); - assert!(decoded.deleted_at.is_some()); - - let deadline_key = backend - .db - .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) - .await - .unwrap(); - assert!(deadline_key.is_none()); - } - - #[tokio::test] - async fn stream_doe_skips_recent_tail_write() { - let backend = test_backend().await; - let basin = BasinName::from_str("doe-basin-recent").unwrap(); - let stream = StreamName::from_str("doe-stream-recent").unwrap(); - let stream_id = seed_stream_with_meta( - &backend, - &basin, - &stream, - stream_meta_with_doe_min_age(MIN_AGE), - ) - .await; - let write_timestamp = put_tail_position( - &backend, - stream_id, - StreamPosition { - seq_num: 1, - timestamp: 1234, - }, - ) - .await; - let deadline = write_timestamp; - - backend - .db - .put( - kv::stream_doe_deadline::ser_key(deadline, stream_id), - kv::stream_doe_deadline::ser_value(MIN_AGE), - ) - .await - .unwrap(); - - process_pending_stream_doe_at(&backend, stream_id, deadline).await; - - let meta = backend - .db - .get(kv::stream_meta::ser_key(&basin, &stream)) - .await - .unwrap() - .expect("stream meta should remain"); - let decoded = kv::stream_meta::deser_value(meta).unwrap(); - assert!(decoded.deleted_at.is_none()); - - let deadline_key = backend - .db - .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) - .await - .unwrap(); - assert!(deadline_key.is_none()); - } - - #[tokio::test] - async fn stream_doe_skips_stream_with_records() { - let backend = test_backend().await; - let basin = BasinName::from_str("doe-basin-nonempty").unwrap(); - let stream = StreamName::from_str("doe-stream-nonempty").unwrap(); - let stream_id = seed_stream_with_meta( - &backend, - &basin, - &stream, - stream_meta_with_doe_min_age(MIN_AGE), - ) - .await; - let deadline = TimestampSecs::now(); - - let pos = StreamPosition { - seq_num: 1, - timestamp: 1234, - }; - backend - .db - .put( - kv::stream_record_timestamp::ser_key(stream_id, pos), - kv::stream_record_timestamp::ser_value(), - ) - .await - .unwrap(); - backend - .db - .put( - kv::stream_doe_deadline::ser_key(deadline, stream_id), - kv::stream_doe_deadline::ser_value(MIN_AGE), - ) - .await - .unwrap(); - - let has_more = backend.clone().tick_stream_doe().await.unwrap(); - assert!(!has_more); - - let meta = backend - .db - .get(kv::stream_meta::ser_key(&basin, &stream)) - .await - .unwrap() - .expect("stream meta should remain"); - let decoded = kv::stream_meta::deser_value(meta).unwrap(); - assert!(decoded.deleted_at.is_none()); - - let deadline_key = backend - .db - .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) - .await - .unwrap(); - assert!(deadline_key.is_none()); - - let timestamp_key = backend - .db - .get(kv::stream_record_timestamp::ser_key(stream_id, pos)) - .await - .unwrap(); - assert!(timestamp_key.is_some()); - } - - #[tokio::test] - async fn stream_doe_ignores_future_deadline() { - let backend = test_backend().await; - let basin = BasinName::from_str("doe-basin-future").unwrap(); - let stream = StreamName::from_str("doe-stream-future").unwrap(); - let stream_id = seed_stream_with_meta( - &backend, - &basin, - &stream, - stream_meta_with_doe_min_age(MIN_AGE), - ) - .await; - let deadline = TimestampSecs::after(Duration::from_secs(3600)); - - backend - .db - .put( - kv::stream_doe_deadline::ser_key(deadline, stream_id), - kv::stream_doe_deadline::ser_value(MIN_AGE), - ) - .await - .unwrap(); - - let has_more = backend.clone().tick_stream_doe().await.unwrap(); - assert!(!has_more); - - let meta = backend - .db - .get(kv::stream_meta::ser_key(&basin, &stream)) - .await - .unwrap() - .expect("stream meta should remain"); - let decoded = kv::stream_meta::deser_value(meta).unwrap(); - assert!(decoded.deleted_at.is_none()); - - let deadline_key = backend - .db - .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) - .await - .unwrap(); - assert!(deadline_key.is_some()); - } - - #[tokio::test] - async fn stream_doe_groups_multiple_deadlines() { - let backend = test_backend().await; - let basin = BasinName::from_str("doe-basin-multi").unwrap(); - let stream = StreamName::from_str("doe-stream-multi").unwrap(); - let stream_id = seed_stream_with_meta( - &backend, - &basin, - &stream, - stream_meta_with_doe_min_age(MIN_AGE), - ) - .await; - let far_future = TimestampSecs::after(Duration::from_secs(3600)); - let deadline_a = TimestampSecs::after(Duration::ZERO); - let deadline_b = TimestampSecs::after(Duration::from_secs(1)); - - backend - .db - .put( - kv::stream_doe_deadline::ser_key(deadline_a, stream_id), - kv::stream_doe_deadline::ser_value(MIN_AGE), - ) - .await - .unwrap(); - backend - .db - .put( - kv::stream_doe_deadline::ser_key(deadline_b, stream_id), - kv::stream_doe_deadline::ser_value(MIN_AGE), - ) - .await - .unwrap(); - - let page = backend.list_pending_stream_doe(far_future).await.unwrap(); - assert!(!page.has_more); - assert_eq!(page.values.len(), 1); - let (pending_stream_id, pending) = page.values.into_iter().next().unwrap(); - assert_eq!(pending_stream_id, stream_id); - let mut deadlines: Vec<_> = pending - .entries() - .iter() - .map(|entry| entry.deadline) - .collect(); - deadlines.sort(); - let mut expected = vec![deadline_a, deadline_b]; - expected.sort(); - assert_eq!(deadlines, expected); - - backend - .process_stream_doe(stream_id, pending) - .await - .unwrap(); - - for deadline in [deadline_a, deadline_b] { - let deadline_key = backend - .db - .get(kv::stream_doe_deadline::ser_key(deadline, stream_id)) - .await - .unwrap(); - assert!(deadline_key.is_none()); + keys: Vec, + ) -> Result, StorageError> { + if needs_doe_migration(snapshot, stream_id).await? { + // Initialization and removal remain atomic. Recheck all eligibility + // in the transaction after the snapshot. + self.initialize_doe_from_deadlines(stream_id, keys).await?; + Ok(Vec::new()) + } else { + Ok(keys) } } - #[tokio::test] - async fn stream_doe_uses_latest_eligible_cutoff_across_pending_entries() { - let backend = test_backend().await; - let basin = BasinName::from_str("doe-basin-pairs").unwrap(); - let stream = StreamName::from_str("doe-stream-pairs").unwrap(); - let stream_id = seed_stream_with_meta( - &backend, - &basin, - &stream, - stream_meta_with_doe_min_age(MIN_AGE), - ) - .await; - - let write_timestamp = put_tail_position( - &backend, - stream_id, - StreamPosition { - seq_num: 1, - timestamp: 1234, - }, - ) - .await; - - backend - .process_stream_doe( - stream_id, - PendingDoeBatch::new(vec![ - kv::stream_doe_deadline::Entry { - deadline: deadline_after(write_timestamp, Duration::from_secs(50)), - min_age: Duration::from_secs(100), - }, - kv::stream_doe_deadline::Entry { - deadline: deadline_after(write_timestamp, Duration::from_secs(100)), - min_age: Duration::from_secs(10), - }, - ]), + async fn initialize_doe_from_deadlines( + &self, + stream_id: StreamId, + keys: Vec, + ) -> Result<(), StorageError> { + let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; + let mapping = db_txn_get_with(&txn, kv::stream_id_mapping::ser_key(stream_id), |entry| { + Ok((kv::stream_id_mapping::deser_value(entry.value)?, entry.seq)) + }) + .await?; + if let Some(((basin, stream), creation_seq)) = mapping + && let Some(meta) = db_txn_get( + &txn, + kv::stream_meta::ser_key(&basin, &stream), + kv::stream_meta::deser_value, ) - .await - .unwrap(); - - let meta = backend - .db - .get(kv::stream_meta::ser_key(&basin, &stream)) - .await - .unwrap() - .expect("stream meta should remain"); - let decoded = kv::stream_meta::deser_value(meta).unwrap(); - assert!(decoded.deleted_at.is_some()); - } - - #[tokio::test] - async fn reconfigure_enabling_doe_on_nonempty_retained_stream_arms_future_deadline() { - let backend = test_backend().await; - let basin = BasinName::from_str("doe-basin-enable").unwrap(); - let stream = StreamName::from_str("doe-stream-enable").unwrap(); - let config = OptionalStreamConfig { - retention_policy: Some(RetentionPolicy::Age(Duration::from_secs(120))), - ..Default::default() - }; - let stream_id = seed_stream_with_meta( - &backend, - &basin, - &stream, - stream_meta_with_config(config, OffsetDateTime::now_utc()), - ) - .await; - let pos = StreamPosition { - seq_num: 1, - timestamp: 1234, - }; - backend - .db - .put( - kv::stream_tail_position::ser_key(stream_id), - kv::stream_tail_position::ser_value(pos), - ) - .await - .unwrap(); - backend - .db - .put( - kv::stream_record_timestamp::ser_key(stream_id, pos), - kv::stream_record_timestamp::ser_value(), - ) - .await - .unwrap(); - - let min_age = Duration::from_secs(30); - let expected_delay = - crate::backend::streamer::doe_arm_delay(Duration::from_secs(120), min_age); - let lower_bound = TimestampSecs::now(); - backend - .reconfigure_stream( - basin, - stream, - StreamReconfiguration { - delete_on_empty: Maybe::from(Some(DeleteOnEmptyReconfiguration { - min_age: Maybe::from(Some(min_age)), - })), - ..Default::default() - }, - ) - .await - .unwrap(); - let upper_bound = TimestampSecs::now(); - - let entries = list_doe_entries(&backend).await; - assert_eq!(entries.len(), 1); - let (deadline, scheduled_stream_id, scheduled_min_age) = entries[0]; - assert_eq!(scheduled_stream_id, stream_id); - assert_eq!(scheduled_min_age, min_age); - - let lower_secs = u64::from(lower_bound.as_u32()).saturating_add(expected_delay.as_secs()); - let upper_secs = u64::from(upper_bound.as_u32()).saturating_add(expected_delay.as_secs()); - let deadline_secs = u64::from(deadline.as_u32()); - assert!(lower_secs <= deadline_secs); - assert!(deadline_secs <= upper_secs); - } - - #[tokio::test] - async fn reconfigure_changing_enabled_doe_does_not_arm_new_deadline() { - let backend = test_backend().await; - let basin = BasinName::from_str("doe-basin-stale").unwrap(); - let stream = StreamName::from_str("doe-stream-stale").unwrap(); - let initial_min_age = Duration::from_secs(10); - let mut config = OptionalStreamConfig::default(); - config.delete_on_empty.min_age = Some(initial_min_age); - let stream_id = seed_stream_with_meta( - &backend, - &basin, - &stream, - stream_meta_with_config(config, OffsetDateTime::now_utc()), - ) - .await; - let existing_deadline = TimestampSecs::from_secs(4_242); - backend - .db - .put( - kv::stream_doe_deadline::ser_key(existing_deadline, stream_id), - kv::stream_doe_deadline::ser_value(initial_min_age), - ) - .await - .unwrap(); - - backend - .reconfigure_stream( - basin, - stream, - StreamReconfiguration { - delete_on_empty: Maybe::from(Some(DeleteOnEmptyReconfiguration { - min_age: Maybe::from(Some(Duration::from_secs(600))), - })), - ..Default::default() - }, - ) - .await - .unwrap(); - - let entries = list_doe_entries(&backend).await; - assert_eq!( - entries, - vec![(existing_deadline, stream_id, initial_min_age)] - ); + .await? + && meta.deleted_at.is_none() + && meta.config.delete_on_empty.min_age().is_some() + { + let state = db_txn_get_with(&txn, kv::stream_doe_state::ser_key(stream_id), |entry| { + Ok((kv::stream_doe_state::deser_value(entry.value)?, entry.seq)) + }) + .await?; + if state.is_none_or(|(_, seq)| seq < creation_seq) { + let next = kv::stream_doe_state::State::Scheduled(kv::stream_doe_state::Check { + at: TimestampSecs::now(), + id: rand::random(), + }); + doe::replace(&txn, stream_id, state.map(|(state, _)| state), Some(next))?; + } + // Existing state, including Parked, is deliberately untouched. + } + for key in keys { + txn.delete(key)?; + } + db_txn_commit_durable(txn).await?; + Ok(()) + } +} + +async fn needs_doe_migration( + snapshot: &slatedb::DbSnapshot, + stream_id: StreamId, +) -> Result { + let Some(((basin, stream), creation_seq)) = db_snapshot_get_with( + snapshot, + kv::stream_id_mapping::ser_key(stream_id), + |entry| Ok((kv::stream_id_mapping::deser_value(entry.value)?, entry.seq)), + ) + .await? + else { + return Ok(false); + }; + let Some(meta) = db_snapshot_get_with( + snapshot, + kv::stream_meta::ser_key(&basin, &stream), + |entry| kv::stream_meta::deser_value(entry.value), + ) + .await? + else { + return Ok(false); + }; + if meta.deleted_at.is_some() || meta.config.delete_on_empty.min_age().is_none() { + return Ok(false); + } + let state_seq = db_snapshot_get_with( + snapshot, + kv::stream_doe_state::ser_key(stream_id), + |entry| { + kv::stream_doe_state::deser_value(entry.value)?; + Ok(entry.seq) + }, + ) + .await?; + Ok(state_seq.is_none_or(|seq| seq < creation_seq)) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use s2_common::{ + config::{ + DeleteOnEmptyReconfiguration, OptionalStreamConfig, RetentionPolicy, + StreamReconfiguration, + }, + maybe::Maybe, + record::{CommandRecord, MeteredExt as _, Record}, + resources::ProvisionMode, + stream::{AppendInput, AppendRecord, AppendRecordParts}, + }; + + use super::*; + use crate::backend::{ + bgtasks::{run_tick, tests::test_backend}, + test_util::{DbWriteTestExt as _, create_stream}, + }; + + fn config(min_age: u64, retention: RetentionPolicy) -> OptionalStreamConfig { + let mut config = OptionalStreamConfig { + retention_policy: Some(retention), + ..Default::default() + }; + config.delete_on_empty.min_age = Some(Duration::from_secs(min_age)); + config + } + + async fn state( + backend: &Backend, + stream_id: StreamId, + ) -> Option<(kv::stream_doe_state::State, u64)> { + backend + .db_get_with(kv::stream_doe_state::ser_key(stream_id), |entry| { + Ok((kv::stream_doe_state::deser_value(entry.value)?, entry.seq)) + }) + .await + .unwrap() + } + + async fn scheduled(backend: &Backend, stream_id: StreamId) -> kv::stream_doe_state::Check { + let Some((kv::stream_doe_state::State::Scheduled(check), _)) = + state(backend, stream_id).await + else { + panic!("expected a scheduled check"); + }; + let checks = backend + .list_pending_stream_doe(TimestampSecs::MAX) + .await + .unwrap(); + assert_eq!(checks.values.len(), 1); + assert_eq!(checks.values[0].stream_id, stream_id); + assert_eq!(checks.values[0].check, check); + check + } + + async fn due(backend: &Backend, stream_id: StreamId) -> PendingCheck { + let txn = backend + .db + .begin(IsolationLevel::SerializableSnapshot) + .await + .unwrap(); + doe::schedule(&txn, stream_id, TimestampSecs::ZERO) + .await + .unwrap(); + db_txn_commit_durable(txn).await.unwrap(); + PendingCheck { + stream_id, + check: scheduled(backend, stream_id).await, + } + } + + async fn append(backend: &Backend, basin: &BasinName, stream: &StreamName, record: Record) { + let record: AppendRecord = AppendRecordParts { + timestamp: None, + record: record.metered(), + } + .try_into() + .unwrap(); + backend + .open_for_append(basin, stream, None, OptionalStreamConfig::default()) + .await + .unwrap() + .append(AppendInput { + records: vec![record].try_into().unwrap(), + match_seq_num: None, + fencing_token: None, + }) + .await + .unwrap(); + } + + fn record() -> Record { + Record::try_from_parts(vec![], Bytes::from_static(b"live")).unwrap() + } + + async fn configure_min_age( + backend: &Backend, + basin: &BasinName, + stream: &StreamName, + min_age: u64, + via_ensure: bool, + ) { + let min_age = Duration::from_secs(min_age); + if via_ensure { + let mut config = backend + .get_stream_config(basin.clone(), stream.clone()) + .await + .unwrap(); + config.delete_on_empty.min_age = min_age; + backend + .provision_stream( + basin.clone(), + stream.clone(), + config.into(), + ProvisionMode::Ensure, + ) + .await + .unwrap(); + } else { + backend + .reconfigure_stream( + basin.clone(), + stream.clone(), + StreamReconfiguration { + delete_on_empty: Maybe::from(Some(DeleteOnEmptyReconfiguration { + min_age: Maybe::from(Some(min_age)), + })), + ..Default::default() + }, + ) + .await + .unwrap(); + } + } + + async fn configure_retention( + backend: &Backend, + basin: &BasinName, + stream: &StreamName, + age: u64, + ) { + backend + .reconfigure_stream( + basin.clone(), + stream.clone(), + StreamReconfiguration { + retention_policy: Maybe::from(Some(RetentionPolicy::Age(Duration::from_secs( + age, + )))), + ..Default::default() + }, + ) + .await + .unwrap(); + } + + async fn legacy(backend: &Backend, stream_id: StreamId, id: Option) -> Bytes { + let key = kv::stream_doe_deadline::ser_key(TimestampSecs::MAX, stream_id, id); + // Migration must not interpret the old min_age, even for future keys. + backend.db.put(&key, [0xff]).assert_durable().await; + key + } + + #[tokio::test] + async fn creation_schedules_once_and_appends_do_not_write_schedules() { + let backend = test_backend().await; + let before = TimestampSecs::after(Duration::from_secs(60)); + let (basin, stream) = + create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; + let stream_id = StreamId::new(&basin, &stream); + let check = scheduled(&backend, stream_id).await; + assert!(check.at >= before); + assert!(check.at <= TimestampSecs::after(Duration::from_secs(60))); + let original = state(&backend, stream_id).await; + append(&backend, &basin, &stream, record()).await; + assert_eq!(state(&backend, stream_id).await, original); + assert_eq!(scheduled(&backend, stream_id).await, check); + let mut old = backend + .db + .scan(kv::key_type_range(kv::KeyType::StreamDeleteOnEmptyDeadline)) + .await + .unwrap(); + assert!(old.next().await.unwrap().is_none()); + backend.close().await.unwrap(); + } + + #[rstest::rstest] + #[case::increase(3600, true)] + #[case::decrease(10, false)] + #[tokio::test] + async fn retry_uses_record_expiration_across_retention_changes( + #[case] new_age: u64, + #[case] append_after: bool, + ) { + let backend = test_backend().await; + let initial_age = if append_after { 10 } else { 3600 }; + let (basin, stream) = create_stream( + &backend, + config(60, RetentionPolicy::Age(Duration::from_secs(initial_age))), + ) + .await; + let stream_id = StreamId::new(&basin, &stream); + let lower_bound = TimestampSecs::after(Duration::from_secs(3600)); + append(&backend, &basin, &stream, record()).await; + let original = state(&backend, stream_id).await; + configure_retention(&backend, &basin, &stream, new_age).await; + if append_after { + append(&backend, &basin, &stream, record()).await; + } + assert_eq!(state(&backend, stream_id).await, original); + let pending = due(&backend, stream_id).await; + backend.process_stream_doe(pending).await.unwrap(); + let replacement = scheduled(&backend, stream_id).await; + assert_ne!(replacement.id, pending.check.id); + assert!( + replacement.at >= lower_bound, + "must preserve a check for the actual stored expiration" + ); + backend.close().await.unwrap(); + } + + #[rstest::rstest] + #[case::never_written(false)] + #[case::expired_records(true)] + #[tokio::test] + async fn empty_old_stream_is_deleted(#[case] written: bool) { + let backend = test_backend().await; + let (basin, stream) = create_stream( + &backend, + config(1, RetentionPolicy::Age(Duration::from_secs(1))), + ) + .await; + let stream_id = StreamId::new(&basin, &stream); + if written { + append(&backend, &basin, &stream, record()).await; + } + tokio::time::sleep(Duration::from_millis(1100)).await; + backend.clone().tick_stream_doe().await.unwrap(); + let meta = backend + .db_get( + kv::stream_meta::ser_key(&basin, &stream), + kv::stream_meta::deser_value, + ) + .await + .unwrap() + .unwrap(); + assert!(meta.deleted_at.is_some()); + assert!(state(&backend, stream_id).await.is_none()); + assert!( + backend + .list_pending_stream_doe(TimestampSecs::MAX) + .await + .unwrap() + .values + .is_empty() + ); + backend.clone().tick_stream_trim().await.unwrap(); + assert!( + backend + .db_get( + kv::stream_id_mapping::ser_key(stream_id), + kv::stream_id_mapping::deser_value + ) + .await + .unwrap() + .is_none() + ); + backend.close().await.unwrap(); + } + + #[rstest::rstest] + #[case::increase(3600)] + #[case::decrease(30)] + #[case::unchanged(60)] + #[case::disable(0)] + #[case::unrepresentable_cutoff(u64::MAX)] + #[tokio::test] + async fn configuration_coalesces_or_removes_the_schedule( + #[case] age: u64, + #[values(false, true)] via_ensure: bool, + ) { + let backend = test_backend().await; + let (basin, stream) = + create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; + let stream_id = StreamId::new(&basin, &stream); + let original = state(&backend, stream_id).await; + configure_min_age(&backend, &basin, &stream, age, via_ensure).await; + if age == 0 { + assert!(state(&backend, stream_id).await.is_none()); + assert!( + backend + .list_pending_stream_doe(TimestampSecs::MAX) + .await + .unwrap() + .values + .is_empty() + ); + configure_min_age(&backend, &basin, &stream, 60, via_ensure).await; + assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); + } else if age == 60 { + assert_eq!(state(&backend, stream_id).await, original); + } else { + assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); + backend.clone().tick_stream_doe().await.unwrap(); + assert!( + scheduled(&backend, stream_id).await.at + >= TimestampSecs::now() + .saturating_add_duration(Duration::from_secs(age.saturating_sub(1))) + ); + assert!(backend.get_stream_config(basin, stream).await.is_ok()); + } + backend.close().await.unwrap(); + } + + #[tokio::test] + async fn partial_trim_wakes_parked_stream_with_finite_records_remaining() { + let backend = test_backend().await; + let (basin, stream) = + create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; + let stream_id = StreamId::new(&basin, &stream); + append(&backend, &basin, &stream, record()).await; + backend + .process_stream_doe(due(&backend, stream_id).await) + .await + .unwrap(); + let parked = state(&backend, stream_id).await; + assert!(matches!( + parked, + Some((kv::stream_doe_state::State::Parked, _)) + )); + assert!( + backend + .list_pending_stream_doe(TimestampSecs::MAX) + .await + .unwrap() + .values + .is_empty() + ); + configure_retention(&backend, &basin, &stream, 3600).await; + append(&backend, &basin, &stream, record()).await; + assert_eq!(state(&backend, stream_id).await, parked); + append( + &backend, + &basin, + &stream, + Record::Command(CommandRecord::Trim(1)), + ) + .await; + backend.clone().tick_stream_trim().await.unwrap(); + assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); + backend.clone().tick_stream_doe().await.unwrap(); + assert!( + scheduled(&backend, stream_id).await.at + > TimestampSecs::after(Duration::from_secs(3500)) + ); + backend.close().await.unwrap(); + } + + #[rstest::rstest] + #[case::disabled(0, false)] + #[case::disabled_before_cleanup(60, false)] + #[case::legacy_only(60, true)] + #[tokio::test] + async fn trim_initializes_missing_state_only_for_enabled_empty_streams( + #[case] min_age: u64, + #[case] legacy_only: bool, + #[values(false, true)] full_trim: bool, + ) { + let backend = test_backend().await; + let (basin, stream) = + create_stream(&backend, config(min_age, RetentionPolicy::Infinite())).await; + let stream_id = StreamId::new(&basin, &stream); + append(&backend, &basin, &stream, record()).await; + append( + &backend, + &basin, + &stream, + Record::Command(CommandRecord::Trim(if full_trim { u64::MAX } else { 1 })), + ) + .await; + if legacy_only { + // Model an enabled stream that has not migrated yet. + let txn = backend + .db + .begin(IsolationLevel::SerializableSnapshot) + .await + .unwrap(); + doe::clear(&txn, stream_id).await.unwrap(); + db_txn_commit_durable(txn).await.unwrap(); + legacy(&backend, stream_id, None).await; + } else if min_age != 0 { + configure_min_age(&backend, &basin, &stream, 0, false).await; + } + assert!(state(&backend, stream_id).await.is_none()); + + backend.clone().tick_stream_trim().await.unwrap(); + let after_trim = state(&backend, stream_id).await; + if legacy_only && full_trim { + assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); + } else { + assert!(after_trim.is_none()); + assert!( + backend + .list_pending_stream_doe(TimestampSecs::MAX) + .await + .unwrap() + .values + .is_empty() + ); + } + + // Migration preserves a full trim's wake or initializes missing state. + if legacy_only { + backend.migrate_stream_doe().await.unwrap(); + if full_trim { + assert_eq!(state(&backend, stream_id).await, after_trim); + } + assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); + } + backend.close().await.unwrap(); + } + + #[tokio::test] + async fn full_trim_restores_doe_without_any_legacy_deadline() { + let backend = test_backend().await; + let (basin, stream) = create_stream(&backend, config(1, RetentionPolicy::Infinite())).await; + let stream_id = StreamId::new(&basin, &stream); + append(&backend, &basin, &stream, record()).await; + // Before upgrade, an infinite-retention stream normally loses its last + // deadline when it fires while the stream is nonempty. + let txn = backend + .db + .begin(IsolationLevel::SerializableSnapshot) + .await + .unwrap(); + doe::clear(&txn, stream_id).await.unwrap(); + db_txn_commit_durable(txn).await.unwrap(); + backend.migrate_stream_doe().await.unwrap(); + assert!(state(&backend, stream_id).await.is_none()); + + append( + &backend, + &basin, + &stream, + Record::Command(CommandRecord::Trim(u64::MAX)), + ) + .await; + backend.clone().tick_stream_trim().await.unwrap(); + assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); + tokio::time::sleep(Duration::from_millis(1100)).await; + backend.clone().tick_stream_doe().await.unwrap(); + let meta = backend + .db_get( + kv::stream_meta::ser_key(&basin, &stream), + kv::stream_meta::deser_value, + ) + .await + .unwrap() + .unwrap(); + assert!(meta.deleted_at.is_some()); + assert!(state(&backend, stream_id).await.is_none()); + backend.close().await.unwrap(); + } + + #[rstest::rstest] + #[tokio::test] + async fn trim_wake_conflicts_with_concurrent_disablement( + #[values(false, true)] missing_state: bool, + ) { + let backend = test_backend().await; + let (basin, stream) = + create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; + let stream_id = StreamId::new(&basin, &stream); + if missing_state { + let txn = backend + .db + .begin(IsolationLevel::SerializableSnapshot) + .await + .unwrap(); + doe::clear(&txn, stream_id).await.unwrap(); + db_txn_commit_durable(txn).await.unwrap(); + } + let txn = backend + .db + .begin(IsolationLevel::SerializableSnapshot) + .await + .unwrap(); + doe::wake_after_trim(&txn, stream_id, false).await.unwrap(); + configure_min_age(&backend, &basin, &stream, 0, false).await; + let error = StorageError::from(txn.commit().await.unwrap_err()); + assert!(error.is_transaction_conflict()); + assert!(state(&backend, stream_id).await.is_none()); + assert!( + backend + .list_pending_stream_doe(TimestampSecs::MAX) + .await + .unwrap() + .values + .is_empty() + ); + backend.close().await.unwrap(); + } + + #[rstest::rstest] + #[case::parked(TerminalTrimOutcome::Parked)] + #[case::deferred(TerminalTrimOutcome::RetryAt(TimestampSecs::MAX))] + #[tokio::test] + async fn event_retaining_same_ticket_invalidates_inflight_result( + #[values(false, true)] trim: bool, + #[case] outcome: TerminalTrimOutcome, + ) { + let backend = test_backend().await; + let (basin, stream) = + create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; + let stream_id = StreamId::new(&basin, &stream); + append(&backend, &basin, &stream, record()).await; + if trim { + configure_retention(&backend, &basin, &stream, 3600).await; + } + let pending = due(&backend, stream_id).await; + let snapshot = backend.observe_doe_check(pending).await.unwrap().unwrap(); + if trim { + // The worker may already have observed the infinite-retention + // record and decided to park. Subsequent appends do not invalidate + // that observation, but removing the blocker must invalidate it. + append(&backend, &basin, &stream, record()).await; + append( + &backend, + &basin, + &stream, + Record::Command(CommandRecord::Trim(1)), + ) + .await; + backend.clone().tick_stream_trim().await.unwrap(); + } else { + configure_min_age(&backend, &basin, &stream, 120, false).await; + } + assert_eq!(scheduled(&backend, stream_id).await, pending.check); + let woken = state(&backend, stream_id).await; + assert!(woken.unwrap().1 > snapshot.revision); + backend + .finish_doe_check(pending, snapshot, outcome) + .await + .unwrap(); + assert_eq!(state(&backend, stream_id).await, woken); + assert_eq!(scheduled(&backend, stream_id).await, pending.check); + backend.close().await.unwrap(); + } + + #[tokio::test] + async fn obsolete_check_cleanup_preserves_wake_after_snapshot() { + let backend = test_backend().await; + let (basin, stream) = + create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; + let stream_id = StreamId::new(&basin, &stream); + let pending = due(&backend, stream_id).await; + let observed = state(&backend, stream_id).await; + configure_min_age(&backend, &basin, &stream, 120, false).await; + let woken = state(&backend, stream_id).await; + assert!(woken.unwrap().1 > observed.unwrap().1); + assert_eq!(scheduled(&backend, stream_id).await, pending.check); + backend.discard_doe_check(pending, observed).await.unwrap(); + assert_eq!(state(&backend, stream_id).await, woken); + assert_eq!(scheduled(&backend, stream_id).await, pending.check); + backend.close().await.unwrap(); + } + + #[tokio::test] + async fn legacy_deadlines_only_initialize_missing_state() { + let backend = test_backend().await; + let (basin, stream) = + create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; + let stream_id = StreamId::new(&basin, &stream); + let txn = backend + .db + .begin(IsolationLevel::SerializableSnapshot) + .await + .unwrap(); + doe::clear(&txn, stream_id).await.unwrap(); + db_txn_commit_durable(txn).await.unwrap(); + let old = legacy(&backend, stream_id, None).await; + let unique = legacy(&backend, stream_id, Some(42)).await; + backend.migrate_stream_doe().await.unwrap(); + assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); + assert!(backend.db.get(&old).await.unwrap().is_none()); + assert!(backend.db.get(&unique).await.unwrap().is_none()); + append(&backend, &basin, &stream, record()).await; + backend.clone().tick_stream_doe().await.unwrap(); + let parked = state(&backend, stream_id).await; + assert!(matches!( + parked, + Some((kv::stream_doe_state::State::Parked, _)) + )); + legacy(&backend, stream_id, Some(43)).await; + backend.migrate_stream_doe().await.unwrap(); + assert_eq!(state(&backend, stream_id).await, parked); + assert!( + backend + .list_pending_stream_doe(TimestampSecs::MAX) + .await + .unwrap() + .values + .is_empty() + ); + backend.close().await.unwrap(); + } + + #[tokio::test] + async fn migration_drains_pages_without_rewriting_schedules() { + let backend = test_backend().await; + let (basin, stream) = + create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; + let stream_id = StreamId::new(&basin, &stream); + let before = state(&backend, stream_id).await; + let mut batch = slatedb::WriteBatch::new(); + for id in 0..=PENDING_LIST_LIMIT { + batch.put( + kv::stream_doe_deadline::ser_key(TimestampSecs::MAX, stream_id, Some(id as u128)), + [], + ); + } + backend.db.write(batch).assert_durable().await; + run_tick( + "stream-delete-on-empty", + &|backend: &Backend| backend.clone().tick_stream_doe(), + &backend, + ) + .await; + let mut remaining = backend + .db + .scan(kv::key_type_range(kv::KeyType::StreamDeleteOnEmptyDeadline)) + .await + .unwrap(); + assert!(remaining.next().await.unwrap().is_none()); + assert_eq!(state(&backend, stream_id).await, before); + scheduled(&backend, stream_id).await; + backend.close().await.unwrap(); + } + + #[tokio::test] + async fn migration_batches_cleanup_across_streams() { + let backend = test_backend().await; + let (basin, stream) = + create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; + let scheduled_id = StreamId::new(&basin, &stream); + let before = state(&backend, scheduled_id).await; + let disabled: StreamName = "disabled".parse().unwrap(); + let deleted: StreamName = "deleted".parse().unwrap(); + for (stream, min_age) in [(&disabled, 0), (&deleted, 60)] { + backend + .provision_stream( + basin.clone(), + stream.clone(), + config(min_age, RetentionPolicy::Infinite()), + ProvisionMode::Ensure, + ) + .await + .unwrap(); + } + backend + .delete_stream(basin.clone(), deleted.clone()) + .await + .unwrap(); + let disabled_id = StreamId::new(&basin, &disabled); + let deleted_id = StreamId::new(&basin, &deleted); + let missing_id = StreamId::new(&basin, &"missing".parse().unwrap()); + let deleted_state = state(&backend, deleted_id).await; + let mut keys = Vec::new(); + for stream_id in [scheduled_id, disabled_id, deleted_id, missing_id] { + for id in 0..3 { + keys.push(legacy(&backend, stream_id, Some(id)).await); + } + } + let before_seq = backend.db.snapshot().await.unwrap().seq(); + backend.migrate_stream_doe().await.unwrap(); + assert_eq!( + backend.db.snapshot().await.unwrap().seq(), + before_seq + 1, + "all cleanup-only streams should share one durable batch" + ); + for key in keys { + assert!(backend.db.get(key).await.unwrap().is_none()); + } + assert_eq!(state(&backend, scheduled_id).await, before); + assert_eq!(state(&backend, deleted_id).await, deleted_state); + assert!(state(&backend, disabled_id).await.is_none()); + assert!(state(&backend, missing_id).await.is_none()); + backend.close().await.unwrap(); + } + + #[rstest::rstest] + #[tokio::test] + async fn migration_revalidates_configuration_after_snapshot(#[values(0, 120)] min_age: u64) { + let backend = test_backend().await; + let (basin, stream) = + create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; + let stream_id = StreamId::new(&basin, &stream); + let txn = backend + .db + .begin(IsolationLevel::SerializableSnapshot) + .await + .unwrap(); + doe::clear(&txn, stream_id).await.unwrap(); + db_txn_commit_durable(txn).await.unwrap(); + let key = legacy(&backend, stream_id, None).await; + let snapshot = backend.db.snapshot().await.unwrap(); + configure_min_age(&backend, &basin, &stream, min_age, false).await; + let configured = state(&backend, stream_id).await; + backend + .migrate_doe_deadlines(&snapshot, stream_id, vec![key.clone()]) + .await + .unwrap(); + assert_eq!(state(&backend, stream_id).await, configured); + assert!(backend.db.get(key).await.unwrap().is_none()); + backend.close().await.unwrap(); + } + + #[tokio::test] + async fn concurrent_wake_conflicts_with_completion_transaction() { + let backend = test_backend().await; + let (basin, stream) = + create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; + let stream_id = StreamId::new(&basin, &stream); + let pending = due(&backend, stream_id).await; + let txn = backend + .db + .begin(IsolationLevel::SerializableSnapshot) + .await + .unwrap(); + let previous = doe::state(&txn, stream_id).await.unwrap(); + doe::replace( + &txn, + stream_id, + previous, + Some(kv::stream_doe_state::State::Parked), + ) + .unwrap(); + // The same ticket is retained, but the scheduler revision changes. + let wake = backend + .db + .begin(IsolationLevel::SerializableSnapshot) + .await + .unwrap(); + doe::schedule(&wake, stream_id, TimestampSecs::now()) + .await + .unwrap(); + db_txn_commit_durable(wake).await.unwrap(); + let error = txn.commit().await.unwrap_err(); + assert_eq!(error.kind(), slatedb::ErrorKind::Transaction); + assert_eq!(scheduled(&backend, stream_id).await, pending.check); + backend.close().await.unwrap(); + } + + #[tokio::test] + async fn full_trim_wakes_parked_stream_and_waits_for_minimum_age() { + let backend = test_backend().await; + let (basin, stream) = + create_stream(&backend, config(60, RetentionPolicy::Infinite())).await; + let stream_id = StreamId::new(&basin, &stream); + append(&backend, &basin, &stream, record()).await; + backend + .process_stream_doe(due(&backend, stream_id).await) + .await + .unwrap(); + assert!(matches!( + state(&backend, stream_id).await, + Some((kv::stream_doe_state::State::Parked, _)) + )); + append( + &backend, + &basin, + &stream, + Record::Command(CommandRecord::Trim(u64::MAX)), + ) + .await; + backend.clone().tick_stream_trim().await.unwrap(); + assert!(scheduled(&backend, stream_id).await.at <= TimestampSecs::now()); + backend.clone().tick_stream_doe().await.unwrap(); + assert!(scheduled(&backend, stream_id).await.at > TimestampSecs::now()); + let meta = backend + .db_get( + kv::stream_meta::ser_key(&basin, &stream), + kv::stream_meta::deser_value, + ) + .await + .unwrap() + .unwrap(); + assert!(meta.deleted_at.is_none()); + backend.close().await.unwrap(); + } + + #[rstest::rstest] + #[case::deletion_pending(TerminalTrimOutcome::DeletionPending)] + #[case::parked(TerminalTrimOutcome::Parked)] + #[tokio::test] + async fn stale_work_cannot_delete_or_reschedule_recreated_stream( + #[case] outcome: TerminalTrimOutcome, + ) { + let backend = test_backend().await; + let configuration = config(60, RetentionPolicy::Infinite()); + let (basin, stream) = create_stream(&backend, configuration.clone()).await; + let stream_id = StreamId::new(&basin, &stream); + let pending = due(&backend, stream_id).await; + let snapshot = backend.observe_doe_check(pending).await.unwrap().unwrap(); + backend + .delete_stream(basin.clone(), stream.clone()) + .await + .unwrap(); + backend.clone().tick_stream_trim().await.unwrap(); + backend + .provision_stream( + basin.clone(), + stream.clone(), + configuration, + ProvisionMode::Ensure, + ) + .await + .unwrap(); + let recreated = state(&backend, stream_id).await; + let client = backend + .streamer_client_guarded(&basin, &stream) + .await + .unwrap(); + assert_eq!( + client + .terminal_trim(TerminalTrimCondition::DeleteOnEmpty { + expected_stream_creation_seq: snapshot.creation_seq, + expected_config_seq: snapshot.config_seq, + }) + .await + .unwrap(), + TerminalTrimOutcome::Obsolete + ); + backend + .finish_doe_check(pending, snapshot, outcome) + .await + .unwrap(); + backend.process_stream_doe(pending).await.unwrap(); + legacy(&backend, stream_id, None).await; + backend.migrate_stream_doe().await.unwrap(); + assert_eq!(state(&backend, stream_id).await, recreated); + scheduled(&backend, stream_id).await; + backend.close().await.unwrap(); } } diff --git a/lite/src/backend/bgtasks/stream_trim.rs b/lite/src/backend/bgtasks/stream_trim.rs index 2a1f332d..db377d0c 100644 --- a/lite/src/backend/bgtasks/stream_trim.rs +++ b/lite/src/backend/bgtasks/stream_trim.rs @@ -3,13 +3,19 @@ use std::ops::RangeTo; use futures::{StreamExt, stream}; use s2_common::{record::NonZeroSeqNum, resources::Page}; use slatedb::{ - WriteBatch, + IsolationLevel, WriteBatch, config::{DurabilityLevel, ScanOptions}, }; use tracing::instrument; +use super::PageProgress; use crate::{ - backend::{Backend, error::StorageError, kv, store::db_txn_get}, + backend::{ + Backend, doe, + error::StorageError, + kv, + store::{db_txn_commit_durable, db_txn_get, db_txn_get_with}, + }, stream_id::StreamId, }; @@ -17,27 +23,34 @@ const PENDING_LIST_LIMIT: usize = 128; const CONCURRENCY: usize = 4; const DELETE_BATCH_SIZE: usize = 10_000; +#[derive(Debug, Clone, Copy)] +struct PendingTrim { + stream_id: StreamId, + trim_point: RangeTo, + /// Commit sequence of the observed marker; bounds the records this job may delete. + marker_seq: u64, +} + impl Backend { pub(super) async fn tick_stream_trim(self) -> Result { let page = self.list_stream_trim_pending().await?; if page.values.is_empty() { return Ok(page.has_more); } + let mut progress = PageProgress::new(page.has_more); let mut processed = stream::iter(page.values) - .map(|(stream_id, trim_point)| { + .map(|pending| { let backend = self.clone(); - async move { backend.process_trim(stream_id, trim_point).await } + async move { backend.process_trim(pending).await } }) .buffer_unordered(CONCURRENCY); while let Some(result) = processed.next().await { - result?; + progress.record_result(result, StorageError::is_transaction_conflict)?; } - Ok(page.has_more) + Ok(progress.should_continue()) } - async fn list_stream_trim_pending( - &self, - ) -> Result)>, StorageError> { + async fn list_stream_trim_pending(&self) -> Result, StorageError> { let scan_opts = ScanOptions { durability_filter: DurabilityLevel::Remote, ..Default::default() @@ -50,7 +63,11 @@ impl Backend { while let Some(kv) = it.next().await? { let stream_id = kv::stream_trim_point::deser_key(kv.key)?; let trim_point = kv::stream_trim_point::deser_value(kv.value)?; - pending.push((stream_id, trim_point)); + pending.push(PendingTrim { + stream_id, + trim_point, + marker_seq: kv.seq, + }); if pending.len() >= PENDING_LIST_LIMIT { return Ok(Page::new(pending, true)); } @@ -58,26 +75,15 @@ impl Backend { Ok(Page::new(pending, false)) } - async fn process_trim( - &self, - stream_id: StreamId, - trim_point: RangeTo, - ) -> Result<(), StorageError> { - let has_remaining_records = self.delete_records(stream_id, trim_point).await?; - if trim_point.end < NonZeroSeqNum::MAX && !has_remaining_records { - self.arm_doe_on_full_trim(stream_id).await?; - } - self.finalize_trim(stream_id, trim_point).await?; - Ok(()) + async fn process_trim(&self, pending: PendingTrim) -> Result<(), StorageError> { + let has_remaining_records = self.delete_records(pending).await?; + self.finalize_trim(pending, has_remaining_records).await } + /// Return whether records remain beyond the trim point. #[instrument(ret, err, skip(self))] - async fn delete_records( - &self, - stream_id: StreamId, - trim_point: RangeTo, - ) -> Result { - let prefix = kv::stream_record_timestamp::ser_key_prefix(stream_id); + async fn delete_records(&self, pending: PendingTrim) -> Result { + let prefix = kv::stream_record_timestamp::ser_key_prefix(pending.stream_id); let scan_opts = ScanOptions { durability_filter: DurabilityLevel::Remote, ..Default::default() @@ -87,67 +93,69 @@ impl Backend { .scan_prefix_with_options(prefix, .., &scan_opts) .await?; let mut batch = WriteBatch::new(); - let mut batch_size = 0usize; + let mut batch_size = 0; let mut has_remaining_records = false; while let Some(kv) = it.next().await? { let (deser_stream_id, pos) = kv::stream_record_timestamp::deser_key(kv.key.clone())?; - debug_assert_eq!(deser_stream_id, stream_id); - if pos.seq_num >= trim_point.end.get() { + debug_assert_eq!(deser_stream_id, pending.stream_id); + // Name reuse requires all old records to be durably deleted first. + // A stale job may see records from the recreated stream; stop before + // deleting records committed after the trim marker it observed. + if pos.seq_num >= pending.trim_point.end.get() || kv.seq > pending.marker_seq { has_remaining_records = true; break; } batch.delete(kv.key); - batch.delete(kv::stream_record_data::ser_key(stream_id, pos)); + batch.delete(kv::stream_record_data::ser_key(pending.stream_id, pos)); batch_size += 1; if batch_size >= DELETE_BATCH_SIZE { - self.db.write(batch).await?; + self.db.write(batch).await?.await_durable().await?; batch = WriteBatch::new(); batch_size = 0; } } - if batch_size > 0 { - self.db.write(batch).await?; + if !batch.is_empty() { + self.db.write(batch).await?.await_durable().await?; } Ok(has_remaining_records) } - #[instrument(ret, err, skip(self))] + #[instrument(skip(self))] async fn finalize_trim( &self, - stream_id: StreamId, - trim_point: RangeTo, + pending: PendingTrim, + has_remaining_records: bool, ) -> Result<(), StorageError> { - let trim_point_key = kv::stream_trim_point::ser_key(stream_id); - let txn = self - .db - .begin(slatedb::IsolationLevel::SerializableSnapshot) - .await?; - let is_full_delete = trim_point == ..NonZeroSeqNum::MAX; - let Some(current_trim_point) = db_txn_get( - &txn, - trim_point_key.clone(), - kv::stream_trim_point::deser_value, - ) - .await? - else { - return Ok(()); - }; - if current_trim_point != trim_point { + let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; + let trim_point_key = kv::stream_trim_point::ser_key(pending.stream_id); + let current_seq = db_txn_get_with(&txn, &trim_point_key, |entry| Ok(entry.seq)).await?; + // A tick can exit on error while its submitted cleanup is still running. + // The next durable scan can capture the old marker before cleanup becomes + // durable. Cleanup removes the old marker and metadata before name reuse, + // but this scanned work can remain queued across recreation. The recreated + // stream's marker may have the same trim point, so compare commit sequences. + if current_seq != Some(pending.marker_seq) { return Ok(()); } + let is_terminal_trim = pending.trim_point == ..NonZeroSeqNum::MAX; txn.delete(trim_point_key)?; - if is_full_delete { - let id_mapping_key = kv::stream_id_mapping::ser_key(stream_id); + if is_terminal_trim { + let id_mapping_key = kv::stream_id_mapping::ser_key(pending.stream_id); if let Some((basin, stream)) = db_txn_get(&txn, &id_mapping_key, kv::stream_id_mapping::deser_value).await? { txn.delete(kv::stream_meta::ser_key(&basin, &stream))?; txn.delete(id_mapping_key)?; } - txn.delete(kv::stream_tail_position::ser_key(stream_id))?; - txn.delete(kv::stream_fencing_token::ser_key(stream_id))?; + txn.delete(kv::stream_tail_position::ser_key(pending.stream_id))?; + txn.delete(kv::stream_fencing_token::ser_key(pending.stream_id))?; + doe::clear(&txn, pending.stream_id).await?; + } else { + // A partial trim may remove the infinite-retention record that + // parked DOE while leaving finite-retention records behind. + doe::wake_after_trim(&txn, pending.stream_id, has_remaining_records).await?; } - txn.commit().await?; + db_txn_commit_durable(txn).await?; Ok(()) } } @@ -170,7 +178,10 @@ mod tests { use time::OffsetDateTime; use super::super::tests::test_backend; - use crate::{backend::kv, stream_id::StreamId}; + use crate::{ + backend::{kv, test_util::DbWriteTestExt as _}, + stream_id::StreamId, + }; fn test_record() -> Metered { let record = Record::try_from_parts(vec![], Bytes::from_static(b"trim-test")).unwrap(); @@ -198,16 +209,16 @@ mod tests { kv::stream_record_data::ser_key(stream_id, pos), kv::stream_record_data::ser_value(metered.as_ref()), ) - .await - .unwrap(); + .assert_durable() + .await; backend .db .put( kv::stream_record_timestamp::ser_key(stream_id, pos), kv::stream_record_timestamp::ser_value(), ) - .await - .unwrap(); + .assert_durable() + .await; } backend @@ -216,8 +227,8 @@ mod tests { kv::stream_trim_point::ser_key(stream_id), kv::stream_trim_point::ser_value(trim_point(3)), ) - .await - .unwrap(); + .assert_durable() + .await; backend.clone().tick_stream_trim().await.unwrap(); @@ -275,16 +286,16 @@ mod tests { kv::stream_meta::ser_key(&basin, &stream), kv::stream_meta::ser_value(&meta), ) - .await - .unwrap(); + .assert_durable() + .await; backend .db .put( kv::stream_id_mapping::ser_key(stream_id), kv::stream_id_mapping::ser_value(&basin, &stream), ) - .await - .unwrap(); + .assert_durable() + .await; backend .db .put( @@ -294,8 +305,8 @@ mod tests { timestamp: 1234, }), ) - .await - .unwrap(); + .assert_durable() + .await; let token = FencingToken::from_str("token-1").unwrap(); backend .db @@ -303,8 +314,8 @@ mod tests { kv::stream_fencing_token::ser_key(stream_id), kv::stream_fencing_token::ser_value(&token), ) - .await - .unwrap(); + .assert_durable() + .await; for seq in 0..3 { let pos = StreamPosition { @@ -317,16 +328,16 @@ mod tests { kv::stream_record_data::ser_key(stream_id, pos), kv::stream_record_data::ser_value(metered.as_ref()), ) - .await - .unwrap(); + .assert_durable() + .await; backend .db .put( kv::stream_record_timestamp::ser_key(stream_id, pos), kv::stream_record_timestamp::ser_value(), ) - .await - .unwrap(); + .assert_durable() + .await; } backend @@ -335,8 +346,8 @@ mod tests { kv::stream_trim_point::ser_key(stream_id), kv::stream_trim_point::ser_value(trim_point(SeqNum::MAX)), ) - .await - .unwrap(); + .assert_durable() + .await; backend.clone().tick_stream_trim().await.unwrap(); @@ -391,35 +402,6 @@ mod tests { } } - #[tokio::test] - async fn stream_trim_skips_stale_trim_point() { - let backend = test_backend().await; - let stream_id: StreamId = [9u8; StreamId::LEN].into(); - - backend - .db - .put( - kv::stream_trim_point::ser_key(stream_id), - kv::stream_trim_point::ser_value(trim_point(10)), - ) - .await - .unwrap(); - - backend - .finalize_trim(stream_id, trim_point(5)) - .await - .unwrap(); - - let current = backend - .db - .get(kv::stream_trim_point::ser_key(stream_id)) - .await - .unwrap() - .expect("trim point should remain"); - let decoded = kv::stream_trim_point::deser_value(current).unwrap(); - assert_eq!(decoded, trim_point(10)); - } - #[tokio::test] async fn stream_trim_paginates_pending_list() { let backend = test_backend().await; @@ -435,7 +417,7 @@ mod tests { kv::stream_trim_point::ser_value(trim_point(1)), ); } - backend.db.write(batch).await.unwrap(); + backend.db.write(batch).assert_durable().await; let has_more = backend.clone().tick_stream_trim().await.unwrap(); assert!(has_more); @@ -472,24 +454,24 @@ mod tests { kv::stream_record_data::ser_key(stream_id, pos), kv::stream_record_data::ser_value(metered.as_ref()), ) - .await - .unwrap(); + .assert_durable() + .await; backend .db .put( kv::stream_record_timestamp::ser_key(stream_id, pos), kv::stream_record_timestamp::ser_value(), ) - .await - .unwrap(); + .assert_durable() + .await; backend .db .put( kv::stream_trim_point::ser_key(stream_id), kv::stream_trim_point::ser_value(trim_point(1)), ) - .await - .unwrap(); + .assert_durable() + .await; backend.clone().tick_stream_trim().await.unwrap(); @@ -514,110 +496,6 @@ mod tests { assert!(trim_point.is_none()); } - #[tokio::test] - async fn stream_trim_does_not_touch_other_streams() { - let backend = test_backend().await; - let stream_id_a: StreamId = [1u8; StreamId::LEN].into(); - let stream_id_b: StreamId = [2u8; StreamId::LEN].into(); - let metered = test_record(); - - for seq in 0..4 { - let pos_a = StreamPosition { - seq_num: seq, - timestamp: 1000 + seq, - }; - backend - .db - .put( - kv::stream_record_data::ser_key(stream_id_a, pos_a), - kv::stream_record_data::ser_value(metered.as_ref()), - ) - .await - .unwrap(); - backend - .db - .put( - kv::stream_record_timestamp::ser_key(stream_id_a, pos_a), - kv::stream_record_timestamp::ser_value(), - ) - .await - .unwrap(); - - let pos_b = StreamPosition { - seq_num: seq, - timestamp: 2000 + seq, - }; - backend - .db - .put( - kv::stream_record_data::ser_key(stream_id_b, pos_b), - kv::stream_record_data::ser_value(metered.as_ref()), - ) - .await - .unwrap(); - backend - .db - .put( - kv::stream_record_timestamp::ser_key(stream_id_b, pos_b), - kv::stream_record_timestamp::ser_value(), - ) - .await - .unwrap(); - } - - backend - .db - .put( - kv::stream_trim_point::ser_key(stream_id_a), - kv::stream_trim_point::ser_value(trim_point(2)), - ) - .await - .unwrap(); - - backend.clone().tick_stream_trim().await.unwrap(); - - for seq in 0..4 { - let pos_a = StreamPosition { - seq_num: seq, - timestamp: 1000 + seq, - }; - let data_a = backend - .db - .get(kv::stream_record_data::ser_key(stream_id_a, pos_a)) - .await - .unwrap(); - let timestamp_a = backend - .db - .get(kv::stream_record_timestamp::ser_key(stream_id_a, pos_a)) - .await - .unwrap(); - if seq < 2 { - assert!(data_a.is_none()); - assert!(timestamp_a.is_none()); - } else { - assert!(data_a.is_some()); - assert!(timestamp_a.is_some()); - } - - let pos_b = StreamPosition { - seq_num: seq, - timestamp: 2000 + seq, - }; - let data_b = backend - .db - .get(kv::stream_record_data::ser_key(stream_id_b, pos_b)) - .await - .unwrap(); - let timestamp_b = backend - .db - .get(kv::stream_record_timestamp::ser_key(stream_id_b, pos_b)) - .await - .unwrap(); - assert!(data_b.is_some()); - assert!(timestamp_b.is_some()); - } - } - #[tokio::test] async fn stream_trim_large_batch_flushes() { let backend = test_backend().await; @@ -645,7 +523,7 @@ mod tests { kv::stream_trim_point::ser_key(stream_id), kv::stream_trim_point::ser_value(trim_point(total)), ); - backend.db.write(batch).await.unwrap(); + backend.db.write(batch).assert_durable().await; backend.clone().tick_stream_trim().await.unwrap(); @@ -678,47 +556,149 @@ mod tests { } #[tokio::test] - async fn finalize_trim_no_trim_point_noop() { - let backend = test_backend().await; - let stream_id: StreamId = [4u8; StreamId::LEN].into(); + async fn stale_deletion_work_preserves_recreated_stream() { + use s2_common::{ + config::OptionalStreamConfig, + resources::ProvisionMode, + stream::{AppendInput, AppendRecord, AppendRecordParts}, + }; + use crate::backend::error::{ProvisionStreamError, ReconfigureStreamError}; + let backend = test_backend().await; + let (basin, stream) = + crate::backend::test_util::create_stream(&backend, OptionalStreamConfig::default()) + .await; + let stream_id = StreamId::new(&basin, &stream); backend - .finalize_trim(stream_id, trim_point(5)) + .open_for_check_tail(&basin, &stream) + .await + .unwrap() + .check_tail() .await .unwrap(); - let trim_point = backend - .db - .get(kv::stream_trim_point::ser_key(stream_id)) + // Pause stream deletion after it submits terminal trim, before it marks metadata. + let mut deletion = Box::pin(backend.delete_stream(basin.clone(), stream.clone())); + assert!(futures::poll!(&mut deletion).is_pending()); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + if backend + .db_get( + kv::stream_trim_point::ser_key(stream_id), + kv::stream_trim_point::deser_value, + ) + .await + .unwrap() + == Some(..NonZeroSeqNum::MAX) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + // A crash here leaves a durable terminal trim with unmarked metadata. + // Provisioning and reconfiguration must reject updates that the pending trim would erase. + assert!(matches!( + backend + .provision_stream( + basin.clone(), + stream.clone(), + OptionalStreamConfig::default(), + ProvisionMode::Ensure + ) + .await, + Err(ProvisionStreamError::StreamDeletionPending(_)) + )); + assert!(matches!( + backend + .reconfigure_stream(basin.clone(), stream.clone(), Default::default()) + .await, + Err(ReconfigureStreamError::StreamDeletionPending(_)) + )); + + let stale_trim = backend + .list_stream_trim_pending() .await + .unwrap() + .values + .pop() .unwrap(); - assert!(trim_point.is_none()); - } - - #[tokio::test] - async fn finalize_trim_clears_matching_trim_point() { - let backend = test_backend().await; - let stream_id: StreamId = [5u8; StreamId::LEN].into(); - + backend.clone().tick_stream_trim().await.unwrap(); backend - .db - .put( - kv::stream_trim_point::ser_key(stream_id), - kv::stream_trim_point::ser_value(trim_point(5)), + .provision_stream( + basin.clone(), + stream.clone(), + OptionalStreamConfig::default(), + ProvisionMode::CreateOnly { + request_token: None, + }, ) .await .unwrap(); - backend - .finalize_trim(stream_id, trim_point(5)) + deletion.await.unwrap(); + let meta = backend + .db_get( + kv::stream_meta::ser_key(&basin, &stream), + kv::stream_meta::deser_value, + ) .await + .unwrap() .unwrap(); + assert!( + meta.deleted_at.is_none(), + "the old deletion request must not mark the recreated stream as deleted" + ); - let trim_point = backend - .db - .get(kv::stream_trim_point::ser_key(stream_id)) + let record: AppendRecord = AppendRecordParts { + timestamp: None, + record: Record::try_from_parts(vec![], Bytes::from_static(b"recreated stream")) + .unwrap() + .metered(), + } + .try_into() + .unwrap(); + let ack = backend + .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) + .await + .unwrap() + .append(AppendInput { + records: vec![record].try_into().unwrap(), + match_seq_num: None, + fencing_token: None, + }) .await .unwrap(); - assert!(trim_point.is_none()); + // A previous tick can have scanned the old marker before cleanup became durable. + backend.process_trim(stale_trim).await.unwrap(); + assert!( + backend + .db_get( + kv::stream_record_data::ser_key(stream_id, ack.start), + kv::stream_record_data::deser_value, + ) + .await + .unwrap() + .is_some(), + "old trim work must not delete records from the recreated stream" + ); + + // Even another terminal marker belongs to different work after recreation. + backend.delete_stream(basin, stream).await.unwrap(); + backend.process_trim(stale_trim).await.unwrap(); + assert_eq!( + backend + .db_get( + kv::stream_trim_point::ser_key(stream_id), + kv::stream_trim_point::deser_value, + ) + .await + .unwrap(), + Some(..NonZeroSeqNum::MAX), + "old trim work must not finalize the recreated stream's deletion" + ); + backend.close().await.unwrap(); } } diff --git a/lite/src/backend/core.rs b/lite/src/backend/core.rs index 0b915877..25359c60 100644 --- a/lite/src/backend/core.rs +++ b/lite/src/backend/core.rs @@ -8,13 +8,12 @@ use futures::{ }; use s2_common::{ basin::BasinName, - config::{BasinConfig, OptionalStreamConfig}, + config::{BasinConfig, OptionalStreamConfig, StreamConfig}, encryption::{EncryptionAlgorithm, EncryptionSpec}, record::{NonZeroSeqNum, SeqNum, StreamPosition}, resources::ProvisionMode, stream::StreamName, }; -use slatedb::config::{DurabilityLevel, ReadOptions, ScanOptions}; use tokio::sync::{Semaphore, broadcast}; use super::{ @@ -26,7 +25,9 @@ use super::{ StreamerMissingInActionError, TransactionConflictError, }, kv, + store::db_snapshot_get_with, streamer::{GuardedStreamerClient, StreamerClient, StreamerGenerationId}, + timestamp::TimestampSecs, }; use crate::{backend::bgtasks::BgtaskTrigger, stream_id::StreamId}; @@ -37,6 +38,7 @@ enum StreamerClientSlot { Initializing { generation_id: StreamerGenerationId, future: StreamerInitFuture, + pending_config: Option<(u64, StreamConfig)>, }, Ready { client: StreamerClient, @@ -112,46 +114,74 @@ impl Backend { basin: BasinName, stream: StreamName, ) -> Result { + // Read one consistent, durable view of the stream. + let snapshot = self.db.snapshot().await.map_err(StorageError::from)?; + self.await_durable_seq(snapshot.seq()).await?; let stream_id = StreamId::new(&basin, &stream); - let (meta, persisted_tail, fencing_token, trim_point) = tokio::try_join!( - self.db_get( + let (meta, persisted_tail, fencing_token, trim_point, stream_creation_seq) = tokio::try_join!( + db_snapshot_get_with( + &snapshot, kv::stream_meta::ser_key(&basin, &stream), - kv::stream_meta::deser_value, + |entry| { Ok((kv::stream_meta::deser_value(entry.value)?, entry.seq)) } ), - self.load_persisted_stream_tail(stream_id), - self.db_get( + db_snapshot_get_with( + &snapshot, + kv::stream_tail_position::ser_key(stream_id), + |entry| { + Ok(( + kv::stream_tail_position::deser_value(entry.value)?, + TimestampSecs::from_millis(entry.create_ts), + )) + } + ), + db_snapshot_get_with( + &snapshot, kv::stream_fencing_token::ser_key(stream_id), - kv::stream_fencing_token::deser_value, + |entry| kv::stream_fencing_token::deser_value(entry.value), ), - self.db_get( + db_snapshot_get_with( + &snapshot, kv::stream_trim_point::ser_key(stream_id), - kv::stream_trim_point::deser_value, - ) + |entry| kv::stream_trim_point::deser_value(entry.value), + ), + db_snapshot_get_with( + &snapshot, + kv::stream_id_mapping::ser_key(stream_id), + |entry| Ok(entry.seq) + ), )?; - let Some(meta) = meta else { + let Some((meta, config_seq)) = meta else { return Err(StreamNotFoundError { basin, stream }.into()); }; let (tail_pos, last_tail_write_timestamp) = - persisted_tail.unwrap_or((StreamPosition::MIN, kv::timestamp::TimestampSecs::ZERO)); + persisted_tail.unwrap_or((StreamPosition::MIN, TimestampSecs::ZERO)); - self.assert_no_records_following_tail(stream_id, &basin, &stream, tail_pos) + if meta.deleted_at.is_some() || trim_point == Some(..NonZeroSeqNum::MAX) { + return Err(StreamDeletionPendingError.into()); + } + + self.assert_no_records_following_tail(&snapshot, stream_id, &basin, &stream, tail_pos) .await?; let fencing_token = fencing_token.unwrap_or_default(); - if trim_point == Some(..NonZeroSeqNum::MAX) { - return Err(StreamDeletionPendingError.into()); - } + let stream_creation_seq = stream_creation_seq.ok_or_else(|| { + StorageError::InvariantViolation(format!( + "live stream `{basin}/{stream}` has no ID mapping" + )) + })?; let streamer_slots = self.streamer_slots.clone(); Ok(super::streamer::Spawner { generation_id, db: self.db.clone(), stream_id, + stream_creation_seq, config: meta.config, + config_seq, cipher: meta.cipher, tail_pos, last_tail_write_timestamp, @@ -168,29 +198,9 @@ impl Backend { })) } - async fn load_persisted_stream_tail( - &self, - stream_id: StreamId, - ) -> Result, StorageError> { - let read_opts = ReadOptions { - durability_filter: DurabilityLevel::Remote, - ..Default::default() - }; - let Some(entry) = self - .db - .get_key_value_with_options(kv::stream_tail_position::ser_key(stream_id), &read_opts) - .await? - else { - return Ok(None); - }; - Ok(Some(( - kv::stream_tail_position::deser_value(entry.value)?, - kv::timestamp::TimestampSecs::from_millis(entry.create_ts), - ))) - } - async fn assert_no_records_following_tail( &self, + snapshot: &slatedb::DbSnapshot, stream_id: StreamId, basin: &BasinName, stream: &StreamName, @@ -201,14 +211,7 @@ impl Backend { seq_num: tail_pos.seq_num, timestamp: 0, }); - let scan_opts = ScanOptions { - durability_filter: DurabilityLevel::Remote, - ..Default::default() - }; - let mut it = self - .db - .scan_prefix_with_options(prefix, start_suffix.., &scan_opts) - .await?; + let mut it = snapshot.scan_prefix(prefix, start_suffix..).await?; let Some(kv) = it.next().await? else { return Ok(()); }; @@ -241,13 +244,22 @@ impl Backend { fn new_initializing_slot(self, basin: &BasinName, stream: &StreamName) -> StreamerClientSlot { let basin = basin.clone(); let stream = stream.clone(); + let stream_id = StreamId::new(&basin, &stream); let generation_id = StreamerGenerationId::next(); - let future = async move { self.start_streamer(generation_id, basin, stream).await } - .boxed() - .shared(); + // Drive initialization independently so cancelled callers cannot leave + // its snapshot pinned in a cached, unpolled future. + let future = tokio::spawn(async move { + let result = self.start_streamer(generation_id, basin, stream).await; + self.streamer_finish_initialization(stream_id, generation_id, &result); + result + }) + .map(|result| result.expect("streamer initialization task panicked")) + .boxed() + .shared(); StreamerClientSlot::Initializing { generation_id, future, + pending_config: None, } } @@ -272,6 +284,13 @@ impl Backend { if client.is_dead() { oe.remove(); } else { + if let StreamerClientSlot::Initializing { + pending_config: Some((seq, config)), + .. + } = oe.get() + { + client.advise_reconfig(*seq, config.clone()); + } oe.insert(StreamerClientSlot::Ready { client: client.clone(), }); @@ -290,30 +309,35 @@ impl Backend { basin: &BasinName, stream: &StreamName, ) -> Result { - let stream_id = StreamId::new(basin, stream); match self.streamer_client_slot(basin, stream) { - StreamerClientSlot::Initializing { - generation_id, - future, - } => { - let result = future.await; - self.streamer_finish_initialization(stream_id, generation_id, &result); - result - } + StreamerClientSlot::Initializing { future, .. } => future.await, StreamerClientSlot::Ready { client } => Ok(client), } } - pub(super) fn streamer_client_if_active( + pub(super) fn advise_stream_config( &self, basin: &BasinName, stream: &StreamName, - ) -> Option { + seq: u64, + config: StreamConfig, + ) { let stream_id = StreamId::new(basin, stream); - let slot = self.streamer_slots.get(&stream_id)?; - match slot.value() { - StreamerClientSlot::Ready { client } if !client.is_dead() => Some(client.clone()), - _ => None, + let Some(mut slot) = self.streamer_slots.get_mut(&stream_id) else { + return; + }; + match slot.value_mut() { + StreamerClientSlot::Ready { client } => { + client.advise_reconfig(seq, config); + } + StreamerClientSlot::Initializing { pending_config, .. } => { + if pending_config + .as_ref() + .is_none_or(|(pending_seq, _)| seq > *pending_seq) + { + *pending_config = Some((seq, config)); + } + } } } @@ -331,11 +355,16 @@ impl Backend { } } + /// Resolve a handle for `stream`, creating it on demand if the basin config allows. + /// + /// `stream_config` is applied over the basin's default stream configuration only if the + /// stream is being created. It must already be validated. pub(super) async fn stream_handle_with_auto_create( &self, basin: &BasinName, stream: &StreamName, - should_auto_create: impl FnOnce(&BasinConfig) -> bool, + auto_create_on: AutoCreateOn, + stream_config: OptionalStreamConfig, resolve_encryption: impl FnOnce(Option) -> Result, ) -> Result where @@ -347,54 +376,66 @@ impl Backend { + From + From, { - match self.streamer_client_guarded(basin, stream).await { - Ok(client) => Ok(StreamHandle { - db: self.db.clone(), - encryption: resolve_encryption(client.cipher())?, - client, - }), + let client = match self.streamer_client_guarded(basin, stream).await { + Ok(client) => client, Err(StreamerError::StreamNotFound(e)) => { let config = match self.get_basin_config(basin.clone()).await { Ok(config) => config, Err(GetBasinConfigError::Storage(e)) => Err(e)?, Err(GetBasinConfigError::BasinNotFound(e)) => Err(e)?, }; - if should_auto_create(&config) { - if let Err(e) = self - .provision_stream( - basin.clone(), - stream.clone(), - OptionalStreamConfig::default(), - ProvisionMode::CreateOnly { - request_token: None, - }, - ) - .await - { - match e { - ProvisionStreamError::Storage(e) => Err(e)?, - ProvisionStreamError::TransactionConflict(e) => Err(e)?, - ProvisionStreamError::BasinDeletionPending(e) => Err(e)?, - ProvisionStreamError::StreamDeletionPending(e) => Err(e)?, - ProvisionStreamError::BasinNotFound(e) => Err(e)?, - ProvisionStreamError::StreamAlreadyExists(_) => {} - ProvisionStreamError::Validation(_) => { - unreachable!("auto-create uses default config") - } + if !auto_create_on.is_enabled(&config) { + return Err(e.into()); + } + if let Err(e) = self + .provision_stream( + basin.clone(), + stream.clone(), + stream_config, + ProvisionMode::CreateOnly { + request_token: None, + }, + ) + .await + { + match e { + ProvisionStreamError::Storage(e) => Err(e)?, + ProvisionStreamError::TransactionConflict(e) => Err(e)?, + ProvisionStreamError::BasinDeletionPending(e) => Err(e)?, + ProvisionStreamError::StreamDeletionPending(e) => Err(e)?, + ProvisionStreamError::BasinNotFound(e) => Err(e)?, + ProvisionStreamError::StreamAlreadyExists(_) => {} + ProvisionStreamError::Validation(e) => { + unreachable!("auto-create config is validated at the API boundary: {e}") } } - let client = self.streamer_client_guarded(basin, stream).await?; - let encryption = resolve_encryption(client.cipher())?; - Ok(StreamHandle { - db: self.db.clone(), - encryption, - client, - }) - } else { - Err(e.into()) } + self.streamer_client_guarded(basin, stream).await? } - Err(e) => Err(e.into()), + Err(e) => return Err(e.into()), + }; + Ok(StreamHandle { + db: self.db.clone(), + encryption: resolve_encryption(client.cipher())?, + client, + }) + } +} + +/// Which basin setting governs creating a missing stream on demand. +#[derive(Debug, Clone, Copy)] +pub(super) enum AutoCreateOn { + /// `create_stream_on_append` + Append, + /// `create_stream_on_read` + Read, +} + +impl AutoCreateOn { + fn is_enabled(self, config: &BasinConfig) -> bool { + match self { + Self::Append => config.create_stream_on_append, + Self::Read => config.create_stream_on_read, } } } @@ -405,15 +446,17 @@ mod tests { use bytes::Bytes; use s2_common::{ - config::{BasinConfig, OptionalStreamConfig, StreamConfig}, + config::{BasinConfig, OptionalStreamConfig, StreamConfig, TimestampingMode}, record::{Metered, MeteredExt as _, Record, StreamPosition}, resources::ProvisionMode, + stream::{AppendInput, AppendRecord, AppendRecordParts}, }; use s2_storage::record::StoredRecord; use slatedb::{WriteBatch, object_store}; use time::OffsetDateTime; use super::*; + use crate::backend::{error::AppendError, test_util::DbWriteTestExt as _}; async fn new_test_backend() -> Backend { let object_store: Arc = @@ -425,6 +468,22 @@ mod tests { Backend::new(db, ByteSize::b(1)) } + fn append_input(body: &str) -> AppendInput { + let record = + Record::try_from_parts(vec![], Bytes::copy_from_slice(body.as_bytes())).unwrap(); + let record: AppendRecord = AppendRecordParts { + timestamp: None, + record: Metered::from(record), + } + .try_into() + .unwrap(); + AppendInput { + records: vec![record].try_into().unwrap(), + match_seq_num: None, + fencing_token: None, + } + } + #[tokio::test] #[should_panic(expected = "invariant violation: stream `testbasin1/stream1` tail_pos")] async fn start_streamer_fails_if_records_exist_after_tail_pos() { @@ -467,7 +526,7 @@ mod tests { kv::stream_record_data::ser_key(stream_id, record_pos), kv::stream_record_data::ser_value(metered_record.as_ref()), ); - backend.db.write(wb).await.unwrap(); + backend.db.write(wb).assert_durable().await; backend .start_streamer(StreamerGenerationId::next(), basin.clone(), stream.clone()) @@ -476,45 +535,77 @@ mod tests { } #[tokio::test] - async fn streamer_client_slot_uses_single_initializer() { - let backend = new_test_backend().await; + async fn streamer_initialization_is_shared_and_survives_cancellation() { + let db = slatedb::Db::builder("test", Arc::new(object_store::memory::InMemory::new())) + .with_settings(slatedb::config::Settings { + flush_interval: None, + ..Default::default() + }) + .build() + .await + .unwrap(); + let backend = Backend::new(db, ByteSize::b(1)); + backend.db.put(b"unflushed", b"value").await.unwrap(); let basin = BasinName::from_str("testbasin2").unwrap(); let stream = StreamName::from_str("stream2").unwrap(); let slot_1 = backend.streamer_client_slot(&basin, &stream); let slot_2 = backend.streamer_client_slot(&basin, &stream); - let (generation_id_1, generation_id_2) = match (slot_1, slot_2) { + let (generation_id_1, generation_id_2, mut future_1, future_2) = match (slot_1, slot_2) { ( StreamerClientSlot::Initializing { generation_id: generation_id_1, + future: future_1, .. }, StreamerClientSlot::Initializing { generation_id: generation_id_2, + future: future_2, .. }, - ) => (generation_id_1, generation_id_2), + ) => (generation_id_1, generation_id_2, future_1, future_2), _ => panic!("expected both slots to be Initializing"), }; assert_eq!(generation_id_1, generation_id_2); assert_eq!(backend.streamer_slots.len(), 1); + + // Cancel every caller while initialization waits for its snapshot to be durable. + assert!(futures::poll!(&mut future_1).is_pending()); + tokio::task::yield_now().await; + drop((future_1, future_2)); + backend.db.flush().await.unwrap(); + + // The missing stream must still finish initialization and release its snapshot. + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while !backend.streamer_slots.is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + backend.close().await.unwrap(); } #[tokio::test] - async fn streamer_client_if_active_is_peek_only() { + async fn config_notification_does_not_start_streamer() { let backend = new_test_backend().await; let basin = BasinName::from_str("testbasin3").unwrap(); let stream = StreamName::from_str("stream3").unwrap(); + backend.advise_stream_config(&basin, &stream, 1, StreamConfig::default()); + assert!(backend.streamer_slots.is_empty()); + backend.close().await.unwrap(); + } + + #[tokio::test] + async fn initializing_streamer_receives_latest_config() { + let backend = new_test_backend().await; + let basin = BasinName::from_str("config-init").unwrap(); + let stream = StreamName::from_str("stream").unwrap(); + let stream_id = StreamId::new(&basin, &stream); backend - .provision_basin( - basin.clone(), - BasinConfig::default(), - ProvisionMode::CreateOnly { - request_token: None, - }, - ) + .provision_basin(basin.clone(), BasinConfig::default(), ProvisionMode::Ensure) .await .unwrap(); backend @@ -522,16 +613,40 @@ mod tests { basin.clone(), stream.clone(), OptionalStreamConfig::default(), - ProvisionMode::CreateOnly { - request_token: None, - }, + ProvisionMode::Ensure, ) .await .unwrap(); - assert!(backend.streamer_slots.is_empty()); - assert!(backend.streamer_client_if_active(&basin, &stream).is_none()); - assert!(backend.streamer_slots.is_empty()); + // Pause initialization after reading metadata, before publishing the client. + let generation_id = StreamerGenerationId::next(); + let client = backend + .start_streamer(generation_id, basin.clone(), stream.clone()) + .await + .unwrap(); + backend.streamer_slots.insert( + stream_id, + StreamerClientSlot::Initializing { + generation_id, + future: futures::future::pending().boxed().shared(), + pending_config: None, + }, + ); + let mut config = StreamConfig::default(); + config.timestamping.mode = TimestampingMode::ClientRequire; + backend.advise_stream_config(&basin, &stream, 20, config); + backend.advise_stream_config(&basin, &stream, 10, StreamConfig::default()); + backend.streamer_finish_initialization(stream_id, generation_id, &Ok(client)); + + let handle = backend + .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) + .await + .unwrap(); + assert!(matches!( + handle.append(append_input("missing timestamp")).await, + Err(AppendError::TimestampMissing(_)) + )); + backend.close().await.unwrap(); } #[tokio::test] @@ -568,6 +683,7 @@ mod tests { StreamerClientSlot::Initializing { generation_id: current_generation_id, future: future.clone(), + pending_config: None, }, ); @@ -587,26 +703,6 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn concurrent_appends_auto_create_stream_without_spurious_not_found() { - use s2_common::stream::{AppendInput, AppendRecord, AppendRecordParts}; - - use crate::backend::error::AppendError; - - fn append_input(body: &str) -> AppendInput { - let record = - Record::try_from_parts(vec![], Bytes::copy_from_slice(body.as_bytes())).unwrap(); - let record: AppendRecord = AppendRecordParts { - timestamp: None, - record: Metered::from(record), - } - .try_into() - .unwrap(); - AppendInput { - records: vec![record].try_into().unwrap(), - match_seq_num: None, - fencing_token: None, - } - } - let backend = new_test_backend().await; let basin = BasinName::from_str("autocreate").unwrap(); backend @@ -631,7 +727,9 @@ mod tests { let basin = basin.clone(); let stream = stream.clone(); tokio::spawn(async move { - let handle = backend.open_for_append(&basin, &stream, None).await?; + let handle = backend + .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) + .await?; handle.append(append_input(&format!("r{i}"))).await }) }) diff --git a/lite/src/backend/doe.rs b/lite/src/backend/doe.rs new file mode 100644 index 00000000..37d7ba7a --- /dev/null +++ b/lite/src/backend/doe.rs @@ -0,0 +1,124 @@ +//! Transactional scheduling shared by configuration changes, trims, and migration. +//! Appends only postpone eligibility and do not need to update the schedule. + +use std::time::Duration; + +use slatedb::DbTransaction; + +use super::{error::StorageError, kv, store::db_txn_get, timestamp::TimestampSecs}; +use crate::stream_id::StreamId; + +/// Bound polling on active streams, while allowing known expirations to defer +/// checks much farther into the future. +pub(super) const RETRY_INTERVAL: Duration = Duration::from_secs(600); + +pub(super) async fn state( + txn: &DbTransaction, + stream_id: StreamId, +) -> Result, StorageError> { + db_txn_get( + txn, + kv::stream_doe_state::ser_key(stream_id), + kv::stream_doe_state::deser_value, + ) + .await +} + +/// Request a check without adding a second ticket. Even when keeping an earlier +/// ticket, rewrite the state to advance its commit sequence: an in-flight worker +/// must not park or postpone the stream after a concurrent trim/configuration wake. +pub(super) async fn schedule( + txn: &DbTransaction, + stream_id: StreamId, + at: TimestampSecs, +) -> Result<(), StorageError> { + let previous = state(txn, stream_id).await?; + schedule_observed(txn, stream_id, previous, at)?; + Ok(()) +} + +/// Most trims need only the state row. A full trim also restores scheduling for +/// older enabled streams whose last legacy deadline was already consumed. +pub(super) async fn wake_after_trim( + txn: &DbTransaction, + stream_id: StreamId, + has_remaining_records: bool, +) -> Result<(), StorageError> { + let previous = state(txn, stream_id).await?; + if previous.is_none() { + if has_remaining_records { + return Ok(()); + } + let Some((basin, stream)) = db_txn_get( + txn, + kv::stream_id_mapping::ser_key(stream_id), + kv::stream_id_mapping::deser_value, + ) + .await? + else { + return Ok(()); + }; + let Some(meta) = db_txn_get( + txn, + kv::stream_meta::ser_key(&basin, &stream), + kv::stream_meta::deser_value, + ) + .await? + else { + return Ok(()); + }; + if meta.deleted_at.is_some() || meta.config.delete_on_empty.min_age().is_none() { + return Ok(()); + } + } + schedule_observed(txn, stream_id, previous, TimestampSecs::now())?; + Ok(()) +} + +fn schedule_observed( + txn: &DbTransaction, + stream_id: StreamId, + previous: Option, + at: TimestampSecs, +) -> Result<(), slatedb::Error> { + let next = match previous { + Some(kv::stream_doe_state::State::Scheduled(check)) if check.at <= at => { + kv::stream_doe_state::State::Scheduled(check) + } + _ => kv::stream_doe_state::State::Scheduled(kv::stream_doe_state::Check { + at, + id: rand::random(), + }), + }; + replace(txn, stream_id, previous, Some(next)) +} + +pub(super) async fn clear(txn: &DbTransaction, stream_id: StreamId) -> Result<(), StorageError> { + let previous = state(txn, stream_id).await?; + replace(txn, stream_id, previous, None)?; + Ok(()) +} + +/// The caller must have read the state in this serializable transaction. +pub(super) fn replace( + txn: &DbTransaction, + stream_id: StreamId, + previous: Option, + next: Option, +) -> Result<(), slatedb::Error> { + if previous != next { + if let Some(kv::stream_doe_state::State::Scheduled(check)) = previous { + txn.delete(kv::stream_doe_check::ser_key(stream_id, check))?; + } + if let Some(kv::stream_doe_state::State::Scheduled(check)) = next { + txn.put(kv::stream_doe_check::ser_key(stream_id, check), [])?; + } + } + let key = kv::stream_doe_state::ser_key(stream_id); + match next { + Some(next) => txn.put(key, kv::stream_doe_state::ser_value(next))?, + None if previous.is_some() => txn.delete(key)?, + None => (), + } + Ok(()) +} diff --git a/lite/src/backend/error.rs b/lite/src/backend/error.rs index 3c441a65..62970c58 100644 --- a/lite/src/backend/error.rs +++ b/lite/src/backend/error.rs @@ -12,12 +12,20 @@ use crate::backend::kv; #[derive(Debug, Clone, thiserror::Error)] pub enum StorageError { + #[error("invariant violation: {0}")] + InvariantViolation(String), #[error("deserialization: {0}")] Deserialization(#[from] kv::DeserializationError), #[error("database: {0}")] Database(Arc), } +impl StorageError { + pub(super) fn is_transaction_conflict(&self) -> bool { + matches!(self, Self::Database(err) if err.kind() == slatedb::ErrorKind::Transaction) + } +} + impl From for StorageError { fn from(error: slatedb::Error) -> Self { StorageError::Database(Arc::new(error)) @@ -106,8 +114,10 @@ pub(super) enum AppendErrorInternal { StreamerMissingInActionError(#[from] StreamerMissingInActionError), #[error(transparent)] RequestDroppedError(#[from] RequestDroppedError), - #[error(transparent)] - StreamDeletionPending(#[from] StreamDeletionPendingError), + #[error("stream deletion pending")] + StreamDeletionPending { + durability_dependency: RangeTo, + }, #[error(transparent)] ConditionFailed(#[from] AppendConditionFailedError), #[error(transparent)] @@ -119,6 +129,9 @@ pub(super) enum AppendErrorInternal { impl AppendErrorInternal { pub fn durability_dependency(&self) -> RangeTo { match self { + Self::StreamDeletionPending { + durability_dependency, + } => *durability_dependency, Self::ConditionFailed(e) => e.durability_dependency(), Self::MaxSeqNum(e) => e.durability_dependency(), _ => ..0, @@ -190,7 +203,9 @@ impl From for AppendError { AppendError::StreamerMissingInActionError(e) } AppendErrorInternal::RequestDroppedError(e) => AppendError::RequestDroppedError(e), - AppendErrorInternal::StreamDeletionPending(e) => AppendError::StreamDeletionPending(e), + AppendErrorInternal::StreamDeletionPending { .. } => { + AppendError::StreamDeletionPending(StreamDeletionPendingError) + } AppendErrorInternal::ConditionFailed(e) => AppendError::ConditionFailed(e), AppendErrorInternal::TimestampMissing(e) => AppendError::TimestampMissing(e), AppendErrorInternal::MaxSeqNum(e) => AppendError::MaxSeqNum(e), @@ -308,7 +323,7 @@ impl From for ListStreamsError { #[derive(Debug, Clone, thiserror::Error)] pub enum ProvisionStreamError { #[error(transparent)] - Storage(#[from] StorageError), + Storage(StorageError), #[error(transparent)] TransactionConflict(#[from] TransactionConflictError), #[error(transparent)] @@ -325,10 +340,16 @@ pub enum ProvisionStreamError { impl From for ProvisionStreamError { fn from(err: slatedb::Error) -> Self { - if err.kind() == slatedb::ErrorKind::Transaction { + Self::from(StorageError::from(err)) + } +} + +impl From for ProvisionStreamError { + fn from(err: StorageError) -> Self { + if err.is_transaction_conflict() { Self::TransactionConflict(TransactionConflictError) } else { - Self::Storage(err.into()) + Self::Storage(err) } } } @@ -392,6 +413,18 @@ pub enum StreamDeleteOnEmptyError { DeleteStream(#[from] DeleteStreamError), } +impl StreamDeleteOnEmptyError { + pub(super) fn is_transaction_conflict(&self) -> bool { + match self { + Self::Storage(err) | Self::DeleteStream(DeleteStreamError::Storage(err)) => { + err.is_transaction_conflict() + } + Self::DeleteStream(DeleteStreamError::TransactionConflict(_)) => true, + _ => false, + } + } +} + #[derive(Debug, Clone, thiserror::Error)] pub enum ListBasinsError { #[error(transparent)] @@ -465,7 +498,7 @@ impl From for ReconfigureBasinError { #[derive(Debug, Clone, thiserror::Error)] pub enum ReconfigureStreamError { #[error(transparent)] - Storage(#[from] StorageError), + Storage(StorageError), #[error(transparent)] TransactionConflict(#[from] TransactionConflictError), #[error(transparent)] @@ -482,10 +515,16 @@ pub enum ReconfigureStreamError { impl From for ReconfigureStreamError { fn from(err: slatedb::Error) -> Self { - if err.kind() == slatedb::ErrorKind::Transaction { + Self::from(StorageError::from(err)) + } +} + +impl From for ReconfigureStreamError { + fn from(err: StorageError) -> Self { + if err.is_transaction_conflict() { Self::TransactionConflict(TransactionConflictError) } else { - Self::Storage(err.into()) + Self::Storage(err) } } } diff --git a/lite/src/backend/kv/mod.rs b/lite/src/backend/kv/mod.rs index 893abe27..95f0e505 100644 --- a/lite/src/backend/kv/mod.rs +++ b/lite/src/backend/kv/mod.rs @@ -1,6 +1,8 @@ pub mod basin_deletion_pending; pub mod basin_meta; +pub mod stream_doe_check; pub mod stream_doe_deadline; +pub mod stream_doe_state; pub mod stream_fencing_token; pub mod stream_id_mapping; pub mod stream_meta; @@ -8,7 +10,6 @@ pub mod stream_record_data; pub mod stream_record_timestamp; pub mod stream_tail_position; pub mod stream_trim_point; -pub mod timestamp; use std::{ops::Range, str::FromStr}; @@ -19,7 +20,7 @@ use s2_common::{ use strum::FromRepr; use thiserror::Error; -use crate::stream_id::StreamId; +use crate::{backend::timestamp::TimestampSecs, stream_id::StreamId}; #[derive(Debug, Clone, Error)] pub enum DeserializationError { @@ -49,6 +50,8 @@ pub enum KeyType { StreamRecordData = 6, StreamRecordTimestamp = 7, StreamDeleteOnEmptyDeadline = 10, + StreamDeleteOnEmptyState = 11, + StreamDeleteOnEmptyCheck = 12, } #[derive(Debug, Clone)] @@ -89,10 +92,21 @@ pub enum Key { /// Key: StreamID Timestamp SeqNum /// Value: empty StreamRecordTimestamp(StreamId, StreamPosition), - /// (SDOED) per-deadline-per-stream, deletable, present while pending - /// Key: TimestampSecs StreamID + /// (SDOED) legacy schedule, consumed only to initialize the new DOE state + /// Key: TimestampSecs StreamID ScheduleID (u128, absent in legacy keys) /// Value: MinAge seconds (u64) - StreamDeleteOnEmptyDeadline(timestamp::TimestampSecs, StreamId), + StreamDeleteOnEmptyDeadline(TimestampSecs, StreamId, Option), + /// (SDOES) per-stream, updatable, optional + /// Key: StreamID + /// Value: Tag (u8: 0 = parked, 1 = scheduled). + /// Scheduled values append TimestampSecs (u32) CheckID (u128). + /// Uses the entry's SlateDB sequence as its revision. + /// State older than the current stream-ID mapping is stale. + StreamDeleteOnEmptyState(StreamId), + /// (SDOEC) per-check, immutable, deletable, time-ordered index of scheduled states + /// Key: TimestampSecs (u32) StreamID CheckID (u128) + /// Value: empty + StreamDeleteOnEmptyCheck(StreamId, stream_doe_state::Check), } impl From for Bytes { @@ -109,8 +123,12 @@ impl From for Bytes { Key::StreamRecordTimestamp(stream_id, pos) => { stream_record_timestamp::ser_key(stream_id, pos) } - Key::StreamDeleteOnEmptyDeadline(deadline, stream_id) => { - stream_doe_deadline::ser_key(deadline, stream_id) + Key::StreamDeleteOnEmptyDeadline(deadline, stream_id, schedule_id) => { + stream_doe_deadline::ser_key(deadline, stream_id, schedule_id) + } + Key::StreamDeleteOnEmptyState(stream_id) => stream_doe_state::ser_key(stream_id), + Key::StreamDeleteOnEmptyCheck(stream_id, check) => { + stream_doe_check::ser_key(stream_id, check) } } } @@ -147,8 +165,16 @@ impl TryFrom for Key { .map(|(stream_id, pos)| Key::StreamRecordData(stream_id, pos)), KeyType::StreamRecordTimestamp => stream_record_timestamp::deser_key(bytes) .map(|(stream_id, pos)| Key::StreamRecordTimestamp(stream_id, pos)), - KeyType::StreamDeleteOnEmptyDeadline => stream_doe_deadline::deser_key(bytes) - .map(|(deadline, stream_id)| Key::StreamDeleteOnEmptyDeadline(deadline, stream_id)), + KeyType::StreamDeleteOnEmptyDeadline => { + stream_doe_deadline::deser_key(bytes).map(|(deadline, stream_id, schedule_id)| { + Key::StreamDeleteOnEmptyDeadline(deadline, stream_id, schedule_id) + }) + } + KeyType::StreamDeleteOnEmptyState => { + stream_doe_state::deser_key(bytes).map(Key::StreamDeleteOnEmptyState) + } + KeyType::StreamDeleteOnEmptyCheck => stream_doe_check::deser_key(bytes) + .map(|(stream_id, check)| Key::StreamDeleteOnEmptyCheck(stream_id, check)), } } } diff --git a/lite/src/backend/kv/stream_doe_check.rs b/lite/src/backend/kv/stream_doe_check.rs new file mode 100644 index 00000000..2ec02c2c --- /dev/null +++ b/lite/src/backend/kv/stream_doe_check.rs @@ -0,0 +1,78 @@ +use std::ops::Range; + +use bytes::{Buf, BufMut, Bytes, BytesMut}; + +use super::{DeserializationError, KeyType, check_exact_size, stream_doe_state::Check}; +use crate::{backend::timestamp::TimestampSecs, stream_id::StreamId}; + +const KEY_LEN: usize = 1 + 4 + StreamId::LEN + 16; + +pub fn ser_key(stream_id: StreamId, check: Check) -> Bytes { + let mut buf = BytesMut::with_capacity(KEY_LEN); + buf.put_u8(KeyType::StreamDeleteOnEmptyCheck as u8); + buf.put_u32(check.at.as_u32()); + buf.put_slice(stream_id.as_bytes()); + buf.put_u128(check.id); + buf.freeze() +} + +pub fn deser_key(mut bytes: Bytes) -> Result<(StreamId, Check), DeserializationError> { + check_exact_size(&bytes, KEY_LEN)?; + let ordinal = bytes.get_u8(); + if ordinal != KeyType::StreamDeleteOnEmptyCheck as u8 { + return Err(DeserializationError::InvalidOrdinal(ordinal)); + } + let at = TimestampSecs::from_secs(bytes.get_u32()); + let mut stream_id = [0; StreamId::LEN]; + bytes.copy_to_slice(&mut stream_id); + let id = bytes.get_u128(); + Ok((stream_id.into(), Check { at, id })) +} + +pub fn due_key_range(now: TimestampSecs) -> Range { + let start = Bytes::from_static(&[KeyType::StreamDeleteOnEmptyCheck as u8]); + let mut end = BytesMut::with_capacity(5); + end.put_u8(KeyType::StreamDeleteOnEmptyCheck as u8); + end.put_u32(now.as_u32()); + start..super::increment_bytes(end).expect("non-empty") +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use super::*; + + proptest! { + #[test] + fn roundtrip_check(at in any::(), id in any::(), stream in any::<[u8; StreamId::LEN]>()) { + let check = Check { at: TimestampSecs::from_secs(at), id }; + let stream_id = StreamId::from(stream); + let key = ser_key(stream_id, check); + prop_assert_eq!(deser_key(key.clone()).unwrap(), (stream_id, check)); + prop_assert_eq!(Bytes::from(super::super::Key::try_from(key.clone()).unwrap()), key); + } + } + + #[test] + fn due_range_includes_every_ticket_at_boundary() { + for seconds in [0, 100, u32::MAX] { + let at = TimestampSecs::from_secs(seconds); + let range = due_key_range(at); + for stream_id in [[0; StreamId::LEN], [u8::MAX; StreamId::LEN]].map(StreamId::from) { + for id in [0, u128::MAX] { + assert!(range.contains(&ser_key(stream_id, Check { at, id }))); + if let Some(next) = seconds.checked_add(1) { + assert!(!range.contains(&ser_key( + stream_id, + Check { + at: TimestampSecs::from_secs(next), + id + } + ))); + } + } + } + } + } +} diff --git a/lite/src/backend/kv/stream_doe_deadline.rs b/lite/src/backend/kv/stream_doe_deadline.rs index 573c5217..8eef06a3 100644 --- a/lite/src/backend/kv/stream_doe_deadline.rs +++ b/lite/src/backend/kv/stream_doe_deadline.rs @@ -1,48 +1,35 @@ -use std::{ops::Range, time::Duration}; - use bytes::{Buf, BufMut, Bytes, BytesMut}; -use super::{DeserializationError, KeyType, check_exact_size, timestamp::TimestampSecs}; -use crate::stream_id::StreamId; +use super::{DeserializationError, KeyType, check_exact_size}; +use crate::{backend::timestamp::TimestampSecs, stream_id::StreamId}; -const KEY_LEN: usize = 1 + 4 + StreamId::LEN; -const VALUE_LEN: usize = 8; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(in crate::backend) struct Entry { - pub deadline: TimestampSecs, - pub min_age: Duration, -} - -impl Entry { - pub fn last_write_cutoff(self) -> Option { - self.deadline.checked_sub_duration(self.min_age) - } -} - -pub fn ser_key(deadline: TimestampSecs, stream_id: StreamId) -> Bytes { - let mut buf = BytesMut::with_capacity(KEY_LEN); +const LEGACY_KEY_LEN: usize = 1 + 4 + StreamId::LEN; +const KEY_LEN: usize = LEGACY_KEY_LEN + 16; +/// Legacy keys are retained only for decoding and migration. New schedules use +/// `stream_doe_state` and `stream_doe_check`. +pub fn ser_key(deadline: TimestampSecs, stream_id: StreamId, schedule_id: Option) -> Bytes { + let key_len = if schedule_id.is_some() { + KEY_LEN + } else { + LEGACY_KEY_LEN + }; + let mut buf = BytesMut::with_capacity(key_len); buf.put_u8(KeyType::StreamDeleteOnEmptyDeadline as u8); buf.put_u32(deadline.as_u32()); buf.put_slice(stream_id.as_bytes()); - debug_assert_eq!(buf.len(), KEY_LEN, "serialized length mismatch"); + if let Some(schedule_id) = schedule_id { + buf.put_u128(schedule_id); + } + debug_assert_eq!(buf.len(), key_len, "serialized length mismatch"); buf.freeze() } -pub fn expired_key_range(deadline: TimestampSecs) -> Range { - let start = Bytes::from(vec![KeyType::StreamDeleteOnEmptyDeadline as u8]); - let end = ser_key_range_end(deadline); - start..end -} - -fn ser_key_range_end(deadline: TimestampSecs) -> Bytes { - let max_stream_id = StreamId::from([u8::MAX; StreamId::LEN]); - let end_key = ser_key(deadline, max_stream_id); - super::increment_bytes(BytesMut::from(end_key.as_ref())).expect("non-empty") -} - -pub fn deser_key(mut bytes: Bytes) -> Result<(TimestampSecs, StreamId), DeserializationError> { - check_exact_size(&bytes, KEY_LEN)?; +pub fn deser_key( + mut bytes: Bytes, +) -> Result<(TimestampSecs, StreamId, Option), DeserializationError> { + if bytes.len() != LEGACY_KEY_LEN { + check_exact_size(&bytes, KEY_LEN)?; + } let ordinal = bytes.get_u8(); if ordinal != (KeyType::StreamDeleteOnEmptyDeadline as u8) { return Err(DeserializationError::InvalidOrdinal(ordinal)); @@ -50,32 +37,20 @@ pub fn deser_key(mut bytes: Bytes) -> Result<(TimestampSecs, StreamId), Deserial let deadline_secs = bytes.get_u32(); let mut stream_id_bytes = [0u8; StreamId::LEN]; bytes.copy_to_slice(&mut stream_id_bytes); + let schedule_id = bytes.has_remaining().then(|| bytes.get_u128()); Ok(( TimestampSecs::from_secs(deadline_secs), stream_id_bytes.into(), + schedule_id, )) } -pub fn ser_value(min_age: Duration) -> Bytes { - let mut buf = BytesMut::with_capacity(VALUE_LEN); - buf.put_u64(min_age.as_secs()); - debug_assert_eq!(buf.len(), VALUE_LEN, "serialized length mismatch"); - buf.freeze() -} - -pub fn deser_value(mut bytes: Bytes) -> Result { - check_exact_size(&bytes, VALUE_LEN)?; - Ok(Duration::from_secs(bytes.get_u64())) -} - #[cfg(test)] mod tests { - use std::time::Duration; - use proptest::prelude::*; use crate::{ - backend::kv::{stream_doe_deadline, timestamp::TimestampSecs}, + backend::{kv::stream_doe_deadline, timestamp::TimestampSecs}, stream_id::StreamId, }; @@ -84,21 +59,17 @@ mod tests { fn roundtrip_stream_doe_deadline_key( deadline_secs in any::(), stream_id_bytes in any::<[u8; StreamId::LEN]>(), + schedule_id in proptest::option::of(any::()), ) { let deadline = TimestampSecs::from_secs(deadline_secs); let stream_id = StreamId::from(stream_id_bytes); - let bytes = stream_doe_deadline::ser_key(deadline, stream_id); - let (decoded_deadline, decoded_stream_id) = stream_doe_deadline::deser_key(bytes).unwrap(); + let bytes = stream_doe_deadline::ser_key(deadline, stream_id, schedule_id); + let (decoded_deadline, decoded_stream_id, decoded_schedule_id) = stream_doe_deadline::deser_key(bytes.clone()).unwrap(); prop_assert_eq!(deadline, decoded_deadline); prop_assert_eq!(stream_id, decoded_stream_id); + prop_assert_eq!(schedule_id, decoded_schedule_id); + let decoded = super::super::Key::try_from(bytes.clone()).unwrap(); + prop_assert_eq!(bytes, bytes::Bytes::from(decoded)); } } - - #[test] - fn roundtrip_stream_doe_deadline_value() { - let min_age = Duration::from_secs(123); - let bytes = stream_doe_deadline::ser_value(min_age); - let decoded = stream_doe_deadline::deser_value(bytes).unwrap(); - assert_eq!(min_age, decoded); - } } diff --git a/lite/src/backend/kv/stream_doe_state.rs b/lite/src/backend/kv/stream_doe_state.rs new file mode 100644 index 00000000..e1f07b49 --- /dev/null +++ b/lite/src/backend/kv/stream_doe_state.rs @@ -0,0 +1,82 @@ +use bytes::{Buf, BufMut, Bytes, BytesMut}; + +use super::{DeserializationError, KeyType, check_exact_size}; +use crate::{backend::timestamp::TimestampSecs, stream_id::StreamId}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Check { + pub at: TimestampSecs, + pub id: u128, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum State { + Scheduled(Check), + /// An observed record has no expiration. Only a trim can remove it. + Parked, +} + +pub fn ser_key(stream_id: StreamId) -> Bytes { + super::ser_stream_id_key(KeyType::StreamDeleteOnEmptyState, stream_id) +} + +pub fn deser_key(bytes: Bytes) -> Result { + super::deser_stream_id_key(KeyType::StreamDeleteOnEmptyState, bytes) +} + +pub fn ser_value(state: State) -> Bytes { + let mut buf = BytesMut::with_capacity(21); + match state { + State::Parked => buf.put_u8(0), + State::Scheduled(Check { at, id }) => { + buf.put_u8(1); + buf.put_u32(at.as_u32()); + buf.put_u128(id); + } + } + buf.freeze() +} + +pub fn deser_value(mut bytes: Bytes) -> Result { + super::check_min_size(&bytes, 1)?; + match bytes.get_u8() { + 0 => { + check_exact_size(&bytes, 0)?; + Ok(State::Parked) + } + 1 => { + check_exact_size(&bytes, 20)?; + Ok(State::Scheduled(Check { + at: TimestampSecs::from_secs(bytes.get_u32()), + id: bytes.get_u128(), + })) + } + ordinal => Err(DeserializationError::InvalidOrdinal(ordinal)), + } +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use super::*; + + proptest! { + #[test] + fn roundtrip_state(at in any::(), id in any::(), stream in any::<[u8; StreamId::LEN]>()) { + for state in [State::Parked, State::Scheduled(Check { at: TimestampSecs::from_secs(at), id })] { + prop_assert_eq!(deser_value(ser_value(state)).unwrap(), state); + } + let key = ser_key(stream.into()); + prop_assert_eq!(deser_key(key.clone()).unwrap(), StreamId::from(stream)); + prop_assert_eq!(Bytes::from(super::super::Key::try_from(key.clone()).unwrap()), key); + } + } + + #[test] + fn reject_truncated_or_unknown_state() { + for bytes in [&[][..], &[0, 1], &[1], &[2]] { + assert!(deser_value(Bytes::copy_from_slice(bytes)).is_err()); + } + } +} diff --git a/lite/src/backend/kv/stream_meta.rs b/lite/src/backend/kv/stream_meta.rs index 70e22a80..f655aaa7 100644 --- a/lite/src/backend/kv/stream_meta.rs +++ b/lite/src/backend/kv/stream_meta.rs @@ -16,6 +16,7 @@ use super::{ DeserializationError, KeyType, check_min_size, deser_json_value, increment_bytes, invalid_value_err, ser_json_value, }; +use crate::backend::resolve_stream_config; const FIELD_SEPARATOR: u8 = b'\0'; @@ -56,12 +57,12 @@ impl TryFrom for StreamMeta { fn try_from(serde: StreamMetaSerde) -> Result { let config = match serde.config { - Some(api_config) => OptionalStreamConfig::try_from(api_config)?.into(), - None => StreamConfig::default(), + Some(api_config) => OptionalStreamConfig::try_from(api_config)?, + None => OptionalStreamConfig::default(), }; Ok(Self { - config, + config: resolve_stream_config(config, OptionalStreamConfig::default()), cipher: serde.cipher, created_at: serde.created_at, deleted_at: serde.deleted_at, @@ -155,7 +156,7 @@ mod tests { use proptest::prelude::*; use s2_common::{ basin::BasinName, - config::{OptionalDeleteOnEmptyConfig, OptionalStreamConfig, StorageClass, StreamConfig}, + config::{OptionalDeleteOnEmptyConfig, OptionalStreamConfig, StreamConfig}, encryption::EncryptionAlgorithm, stream::{StreamName, StreamNamePrefix, StreamNameStartAfter}, }; @@ -167,7 +168,7 @@ mod tests { #[test] fn value_roundtrip_stream_meta() { let config = OptionalStreamConfig { - storage_class: Some(StorageClass::Express), + storage_class: Some("express".into()), delete_on_empty: OptionalDeleteOnEmptyConfig { min_age: Some(Duration::ZERO), }, @@ -228,7 +229,7 @@ mod tests { let decoded = super::deser_value(bytes).unwrap(); let default_config = StreamConfig::default(); - assert_eq!(decoded.config.storage_class, default_config.storage_class); + assert_eq!(decoded.config.storage_class.as_deref(), Some("express")); assert_eq!( decoded.config.retention_policy, default_config.retention_policy diff --git a/lite/src/backend/mod.rs b/lite/src/backend/mod.rs index be8db24f..99aab0fb 100644 --- a/lite/src/backend/mod.rs +++ b/lite/src/backend/mod.rs @@ -1,15 +1,23 @@ -use s2_common::encryption::EncryptionSpec; +use s2_common::{ + config::{OptionalStreamConfig, StreamConfig}, + encryption::EncryptionSpec, +}; pub mod error; mod basins; pub mod bgtasks; mod core; +mod doe; mod durability_notifier; mod read; mod store; mod streamer; mod streams; +mod timestamp; + +#[cfg(test)] +mod test_util; mod append; mod kv; @@ -25,3 +33,12 @@ pub struct StreamHandle { } pub const FOLLOWER_MAX_LAG: usize = 25; + +fn resolve_stream_config( + config: OptionalStreamConfig, + basin_defaults: OptionalStreamConfig, +) -> StreamConfig { + let mut config = config.merge(basin_defaults); + config.storage_class.get_or_insert_with(|| "express".into()); + config +} diff --git a/lite/src/backend/read.rs b/lite/src/backend/read.rs index 7043409f..3ff6c5bc 100644 --- a/lite/src/backend/read.rs +++ b/lite/src/backend/read.rs @@ -4,6 +4,7 @@ use futures::{Stream, StreamExt as _}; use s2_common::{ basin::BasinName, caps, + config::OptionalStreamConfig, encryption::{EncryptionKey, EncryptionSpec}, read_extent::{EvaluatedReadLimit, ReadLimit, ReadUntil}, record::{Metered, MeteredSize as _, SeqNum, StreamPosition, Timestamp}, @@ -15,7 +16,7 @@ use s2_storage::record::{ use slatedb::config::{DurabilityLevel, ScanOptions}; use tokio::{sync::broadcast, time::Instant}; -use super::{Backend, StreamHandle}; +use super::{Backend, StreamHandle, core::AutoCreateOn}; use crate::{ backend::{ error::{ @@ -28,6 +29,7 @@ use crate::{ }; impl Backend { + /// Open a stream for a check tail. pub async fn open_for_check_tail( &self, basin: &BasinName, @@ -36,22 +38,29 @@ impl Backend { self.stream_handle_with_auto_create::( basin, stream, - |config| config.create_stream_on_read, + AutoCreateOn::Read, + OptionalStreamConfig::default(), |_| Ok(EncryptionSpec::Plain), ) .await } + /// Open a stream for a read or read session. + /// + /// `stream_config` is applied if the stream is created on read. Unset fields inherit the + /// basin's default stream configuration. Ignored if the stream already exists. pub async fn open_for_read( &self, basin: &BasinName, stream: &StreamName, encryption_key: Option, + stream_config: OptionalStreamConfig, ) -> Result { self.stream_handle_with_auto_create::( basin, stream, - |config| config.create_stream_on_read, + AutoCreateOn::Read, + stream_config, |cipher| Ok(EncryptionSpec::resolve(cipher, encryption_key)?), ) .await @@ -97,6 +106,10 @@ async fn read_session( start: ReadStart, end: ReadEnd, ) -> Result> + 'static, ReadError> { + // An exhausted limit completes even when the requested start is unwritten. + if end.limit.remaining(0, 0) == EvaluatedReadLimit::Exhausted { + return Ok(futures::stream::empty().left_stream()); + } let stream_id = client.stream_id(); let tail = client.check_tail().await?; let mut state = ReadSessionState { @@ -240,7 +253,7 @@ async fn read_session( } } }; - Ok(session) + Ok(session.right_stream()) } async fn read_start_seq_num( @@ -267,13 +280,9 @@ async fn read_start_seq_num( return Err(UnwrittenError(tail).into()); } } - if let ReadPosition::SeqNum(start_seq_num) = read_pos - && start_seq_num == tail.seq_num - && !end.may_follow() - { - return Err(UnwrittenError(tail).into()); - } - Ok(match read_pos { + // Resolve to a sequence number before deciding whether the read starts at the tail, so a + // timestamp start that resolves to the tail is treated like a sequence number start there. + let start_seq_num = match read_pos { ReadPosition::SeqNum(start_seq_num) => start_seq_num, ReadPosition::Timestamp(start_timestamp) => { resolve_timestamp(db, stream_id, start_timestamp) @@ -281,7 +290,11 @@ async fn read_start_seq_num( .unwrap_or(tail) .seq_num } - }) + }; + if start_seq_num == tail.seq_num && !end.may_follow() { + return Err(UnwrittenError(tail).into()); + } + Ok(start_seq_num) } async fn resolve_timestamp( @@ -421,7 +434,9 @@ mod tests { use super::*; use crate::{ - backend::{FOLLOWER_MAX_LAG, kv, streamer::DORMANT_TIMEOUT}, + backend::{ + FOLLOWER_MAX_LAG, kv, streamer::DORMANT_TIMEOUT, test_util::DbWriteTestExt as _, + }, stream_id::StreamId, }; @@ -496,8 +511,8 @@ mod tests { ), kv::stream_record_timestamp::ser_value(), ) - .await - .unwrap(); + .assert_durable() + .await; backend .db .put( @@ -510,8 +525,8 @@ mod tests { ), kv::stream_record_timestamp::ser_value(), ) - .await - .unwrap(); + .assert_durable() + .await; // Should find record in stream_a let result = resolve_timestamp(&backend.db, stream_a, 500).await.unwrap(); @@ -562,7 +577,7 @@ mod tests { let input = append_input(Record::try_from_parts(vec![], bytes::Bytes::from("x")).unwrap()); let ack = backend - .open_for_append(&basin, &stream, None) + .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) .await .unwrap() .append(input) @@ -573,7 +588,7 @@ mod tests { let stream_id = StreamId::new(&basin, &stream); let mut batch = WriteBatch::new(); batch.delete(kv::stream_record_data::ser_key(stream_id, ack.start)); - backend.db.write(batch).await.unwrap(); + backend.db.write(batch).assert_durable().await; let start = ReadStart { from: ReadFrom::SeqNum(0), @@ -585,7 +600,7 @@ mod tests { wait: None, }; let session = backend - .open_for_read(&basin, &stream, None) + .open_for_read(&basin, &stream, None, OptionalStreamConfig::default()) .await .unwrap() .read(start, end) @@ -642,7 +657,7 @@ mod tests { }; let session = backend - .open_for_read(&basin, &stream, None) + .open_for_read(&basin, &stream, None, OptionalStreamConfig::default()) .await .unwrap() .read(start, end) @@ -716,7 +731,7 @@ mod tests { let initial_input = append_input(Record::try_from_parts(vec![], bytes::Bytes::from("initial")).unwrap()); backend - .open_for_append(&basin, &stream, None) + .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) .await .unwrap() .append(initial_input) @@ -736,7 +751,7 @@ mod tests { }; let session = backend - .open_for_read(&basin, &stream, None) + .open_for_read(&basin, &stream, None, OptionalStreamConfig::default()) .await .unwrap() .read(start, end) @@ -776,7 +791,7 @@ mod tests { let follow_input = append_input(Record::try_from_parts(vec![], bytes::Bytes::from("follow-1")).unwrap()); backend - .open_for_append(&basin, &stream, None) + .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) .await .unwrap() .append(follow_input) @@ -881,7 +896,7 @@ mod tests { wait: Some(wait), }; let session = backend - .open_for_read(&basin, &stream, None) + .open_for_read(&basin, &stream, None, OptionalStreamConfig::default()) .await .unwrap() .read(start, end) @@ -906,7 +921,7 @@ mod tests { Record::try_from_parts(vec![], bytes::Bytes::from(format!("lagged-{i}"))).unwrap(), ); let ack = backend - .open_for_append(&basin, &stream, None) + .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) .await .unwrap() .append(input) @@ -915,7 +930,7 @@ mod tests { delete_batch.delete(kv::stream_record_data::ser_key(stream_id, ack.start)); } - backend.db.write(delete_batch).await.unwrap(); + backend.db.write(delete_batch).assert_durable().await; tokio::time::advance(wait + Duration::from_secs(1)).await; tokio::task::yield_now().await; @@ -960,7 +975,7 @@ mod tests { let initial_input = append_input(Record::try_from_parts(vec![], bytes::Bytes::from("initial")).unwrap()); backend - .open_for_append(&basin, &stream, None) + .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) .await .unwrap() .append(initial_input) @@ -977,7 +992,7 @@ mod tests { wait: None, }; let session = backend - .open_for_read(&basin, &stream, None) + .open_for_read(&basin, &stream, None, OptionalStreamConfig::default()) .await .unwrap() .read(start, end) @@ -1007,7 +1022,7 @@ mod tests { let follow_input = append_input(Record::try_from_parts(vec![], bytes::Bytes::from("follow-1")).unwrap()); backend - .open_for_append(&basin, &stream, None) + .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) .await .unwrap() .append(follow_input) diff --git a/lite/src/backend/store.rs b/lite/src/backend/store.rs index 791bc561..6b26bcdb 100644 --- a/lite/src/backend/store.rs +++ b/lite/src/backend/store.rs @@ -1,6 +1,6 @@ use bytes::Bytes; use slatedb::{ - DbTransaction, + DbSnapshot, DbTransaction, KeyValue, config::{DurabilityLevel, ReadOptions}, }; @@ -19,6 +19,15 @@ impl Backend { &self, key: K, deser: impl FnOnce(Bytes) -> Result, + ) -> Result, StorageError> { + self.db_get_with(key, |entry| deser(entry.value)).await + } + + /// Read a remotely durable row, including its sequence and timestamps. + pub(super) async fn db_get_with + Send, V>( + &self, + key: K, + deser: impl FnOnce(KeyValue) -> Result, ) -> Result, StorageError> { let read_opts = ReadOptions { durability_filter: DurabilityLevel::Remote, @@ -26,7 +35,7 @@ impl Backend { }; let value = self .db - .get_with_options(key, &read_opts) + .get_key_value_with_options(key, &read_opts) .await? .map(deser) .transpose()?; @@ -34,11 +43,40 @@ impl Backend { } } +pub(super) async fn db_snapshot_get_with + Send, V>( + snapshot: &DbSnapshot, + key: K, + deser: impl FnOnce(KeyValue) -> Result, +) -> Result, StorageError> { + Ok(snapshot.get_key_value(key).await?.map(deser).transpose()?) +} + pub(super) async fn db_txn_get + Send, V>( txn: &DbTransaction, key: K, deser: impl FnOnce(Bytes) -> Result, ) -> Result, StorageError> { - let value = txn.get(key).await?.map(deser).transpose()?; + db_txn_get_with(txn, key, |entry| deser(entry.value)).await +} + +/// Read a transaction row, which may not yet be durable. +pub(super) async fn db_txn_get_with + Send, V>( + txn: &DbTransaction, + key: K, + deser: impl FnOnce(KeyValue) -> Result, +) -> Result, StorageError> { + let value = txn.get_key_value(key).await?.map(deser).transpose()?; Ok(value) } + +/// Commit metadata changes and wait until remote reads can observe them. +/// Return the committed sequence number, or `None` if there were no writes. +pub(super) async fn db_txn_commit_durable( + txn: DbTransaction, +) -> Result, slatedb::Error> { + let Some(handle) = txn.commit().await? else { + return Ok(None); + }; + handle.await_durable().await?; + Ok(Some(handle.seqnum())) +} diff --git a/lite/src/backend/streamer.rs b/lite/src/backend/streamer.rs index d8acb6a3..1dd567e8 100644 --- a/lite/src/backend/streamer.rs +++ b/lite/src/backend/streamer.rs @@ -28,7 +28,7 @@ use s2_storage::record::{ }; use slatedb::{ IterationOrder, WriteBatch, - config::{PutOptions, ScanOptions, Ttl, WriteOptions}, + config::{PutOptions, ScanOptions, Ttl}, }; use tokio::{ sync::{Semaphore, SemaphorePermit, broadcast, mpsc, oneshot}, @@ -40,27 +40,21 @@ use crate::{ backend::{ append, bgtasks::BgtaskTrigger, + doe, durability_notifier::DurabilityNotifier, error::{ AppendConditionFailedError, AppendErrorInternal, AppendTimestampRequiredError, DeleteStreamError, MaxSeqNumError, RequestDroppedError, StorageError, - StreamDeletionPendingError, StreamerMissingInActionError, + StreamerMissingInActionError, }, kv, + timestamp::TimestampSecs, }, metrics, stream_id::StreamId, }; pub(super) const DORMANT_TIMEOUT: Duration = Duration::from_secs(60); -// Rate-limit delete-on-empty scheduling and pad deadlines to cover the period. -const DOE_DEADLINE_REFRESH_PERIOD: Duration = Duration::from_secs(600); - -pub(super) fn doe_arm_delay(base_delay: Duration, min_age: Duration) -> Duration { - base_delay - .saturating_add(min_age) - .saturating_add(DOE_DEADLINE_REFRESH_PERIOD) -} #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) struct StreamerGenerationId(u64); @@ -80,7 +74,6 @@ struct InFlightAppend { struct DbSubmitAppendOptions { retention: RetentionPolicy, - doe_deadline: Option, fencing_token: Option, trim_point: Option>, } @@ -210,10 +203,14 @@ pub(super) struct Spawner { pub generation_id: StreamerGenerationId, pub db: slatedb::Db, pub stream_id: StreamId, + /// Database commit sequence that created the stream's ID mapping. + /// Stable across streamer restarts; changes when the stream is recreated. + pub stream_creation_seq: u64, pub config: StreamConfig, + pub config_seq: u64, pub cipher: Option, pub tail_pos: StreamPosition, - pub last_tail_write_timestamp: kv::timestamp::TimestampSecs, + pub last_tail_write_timestamp: TimestampSecs, pub fencing_token: FencingToken, pub trim_point: RangeTo, pub append_inflight_bytes_sema: Arc, @@ -230,7 +227,9 @@ impl Spawner { generation_id, db, stream_id, + stream_creation_seq, config, + config_seq, cipher, tail_pos, last_tail_write_timestamp, @@ -246,8 +245,10 @@ impl Spawner { let streamer = Streamer { db, stream_id, + stream_creation_seq, msg_tx: msg_tx.clone(), config, + config_seq, last_tail_write_timestamp, fencing_token: CommandState { state: fencing_token, @@ -257,7 +258,6 @@ impl Spawner { state: trim_point, applied_point: ..tail_pos.seq_num, }, - last_doe_deadline_at: None, db_writes_pending: VecDeque::new(), db_durability_subscription: 0, inflight_appends: VecDeque::new(), @@ -306,12 +306,13 @@ impl CommandState { struct Streamer { db: slatedb::Db, stream_id: StreamId, + stream_creation_seq: u64, msg_tx: mpsc::UnboundedSender, config: StreamConfig, - last_tail_write_timestamp: kv::timestamp::TimestampSecs, + config_seq: u64, + last_tail_write_timestamp: TimestampSecs, fencing_token: CommandState, trim_point: CommandState>, - last_doe_deadline_at: Option, db_writes_pending: VecDeque>>, db_durability_subscription: u64, inflight_appends: VecDeque, @@ -400,7 +401,10 @@ impl Streamer { return; }; let sequenced_records = if self.trim_point.state.end == SeqNum::MAX { - Err(StreamDeletionPendingError.into()) + Err(AppendErrorInternal::StreamDeletionPending { + // The terminal trim must be durable before reporting deletion pending. + durability_dependency: self.trim_point.applied_point, + }) } else { self.sequence_records(input) }; @@ -422,7 +426,6 @@ impl Streamer { let seq_num_range = first_pos.seq_num..next_pos.seq_num; let opts = DbSubmitAppendOptions { retention: self.config.retention_policy, - doe_deadline: self.doe_deadline_maybe(), fencing_token: self .fencing_token .is_applied_in(&seq_num_range) @@ -437,7 +440,7 @@ impl Streamer { .boxed(), ); self.pending_appends.accept(ticket, first_pos..next_pos); - self.last_tail_write_timestamp = kv::timestamp::TimestampSecs::now(); + self.last_tail_write_timestamp = TimestampSecs::now(); } Err(e) => { self.pending_appends.reject(ticket, e, self.stable_pos); @@ -452,30 +455,41 @@ impl Streamer { ) { match condition { TerminalTrimCondition::Always => { - self.append_terminal_trim(reply_tx); + self.ensure_terminal_trim(reply_tx); } - TerminalTrimCondition::DeleteOnEmpty { last_write_cutoff } => { - if self.last_tail_write_timestamp > last_write_cutoff - || self.next_assignable_pos().seq_num != self.stable_pos.seq_num - || self.config.delete_on_empty.min_age().is_none() - { - let _ = reply_tx.send(Ok(TerminalTrimOutcome::Ineligible)); - return; - } - - let db = self.db.clone(); - let stream_id = self.stream_id; - let stable_pos_snapshot = self.stable_pos; - let msg_tx = self.msg_tx.clone(); - tokio::spawn(async move { - let has_records = stream_has_records(&db, stream_id).await; - let _ = msg_tx.send(Message::DeleteOnEmptyCheckResult { - stable_pos_snapshot, - last_write_cutoff, - has_records, - reply_tx, + TerminalTrimCondition::DeleteOnEmpty { + expected_stream_creation_seq, + expected_config_seq, + } => { + if self.stream_creation_seq != expected_stream_creation_seq { + let _ = reply_tx.send(Ok(TerminalTrimOutcome::Obsolete)); + } else if self.trim_point.state.end == SeqNum::MAX { + self.ensure_terminal_trim(reply_tx); + } else if self.config_seq != expected_config_seq { + // The worker may have observed a configuration commit before + // its notification reached this actor (or vice versa). Do not + // defer using an age that may have just been decreased. + let _ = reply_tx.send(Ok(TerminalTrimOutcome::RetryAt(TimestampSecs::after( + doe::RETRY_INTERVAL, + )))); + } else if self.config.delete_on_empty.min_age().is_none() { + let _ = reply_tx.send(Ok(TerminalTrimOutcome::Obsolete)); + } else { + let db = self.db.clone(); + let stream_id = self.stream_id; + let stable_pos_snapshot = self.stable_pos; + let config_seq_snapshot = self.config_seq; + let msg_tx = self.msg_tx.clone(); + tokio::spawn(async move { + let records = stream_record_presence(&db, stream_id).await; + let _ = msg_tx.send(Message::DeleteOnEmptyCheckResult { + stable_pos_snapshot, + config_seq_snapshot, + records, + reply_tx, + }); }); - }); + } } } } @@ -483,34 +497,66 @@ impl Streamer { fn handle_doe_check_result( &mut self, stable_pos_snapshot: StreamPosition, - last_write_cutoff: kv::timestamp::TimestampSecs, - has_records: Result, + config_seq_snapshot: u64, + records: Result, reply_tx: oneshot::Sender>, ) { - match has_records { - Ok(true) => { - let _ = reply_tx.send(Ok(TerminalTrimOutcome::Ineligible)); - } - Ok(false) => { - if self.trim_point.state.end == SeqNum::MAX { - let _ = reply_tx.send(Ok(TerminalTrimOutcome::DeletionPending)); - } else if self.stable_pos != stable_pos_snapshot - || self.next_assignable_pos() != stable_pos_snapshot - || self.last_tail_write_timestamp > last_write_cutoff - || self.config.delete_on_empty.min_age().is_none() - { - let _ = reply_tx.send(Ok(TerminalTrimOutcome::Ineligible)); - } else { - self.append_terminal_trim(reply_tx); - } - } + let records = match records { + Ok(records) => records, Err(err) => { let _ = reply_tx.send(Err(err.into())); + return; } + }; + if self.trim_point.state.end == SeqNum::MAX { + self.ensure_terminal_trim(reply_tx); + return; } + if self.config_seq != config_seq_snapshot { + let _ = reply_tx.send(Ok(TerminalTrimOutcome::RetryAt(TimestampSecs::after( + doe::RETRY_INTERVAL, + )))); + return; + } + let Some(min_age) = self.config.delete_on_empty.min_age() else { + let _ = reply_tx.send(Ok(TerminalTrimOutcome::Obsolete)); + return; + }; + let outcome = match records { + // Appends cannot remove an observed record, so these bounds remain + // useful even when the tail advances. The scheduler's revision check + // protects against a trim removing the record before we finish. + RecordPresence::ExpiresAt(at) => self.doe_retry_at(at), + RecordPresence::Unbounded => TerminalTrimOutcome::Parked, + RecordPresence::Empty => { + let old_enough = TimestampSecs::now() + .checked_sub_duration(min_age) + .is_some_and(|cutoff| self.last_tail_write_timestamp <= cutoff); + if self.stable_pos == stable_pos_snapshot + && self.next_assignable_pos() == stable_pos_snapshot + && old_enough + { + self.ensure_terminal_trim(reply_tx); + return; + } + self.doe_retry_at(TimestampSecs::ZERO) + } + }; + let _ = reply_tx.send(Ok(outcome)); } - fn append_terminal_trim( + fn doe_retry_at(&self, earliest_empty: TimestampSecs) -> TerminalTrimOutcome { + let age_at = self + .last_tail_write_timestamp + .saturating_add_duration(self.config.delete_on_empty.min_age().unwrap_or_default()); + TerminalTrimOutcome::RetryAt( + TimestampSecs::after(doe::RETRY_INTERVAL) + .max(age_at) + .max(earliest_empty), + ) + } + + fn ensure_terminal_trim( &mut self, reply_tx: oneshot::Sender>, ) { @@ -524,7 +570,7 @@ impl Streamer { tokio::spawn(async move { let result = match append_reply_rx.await { Ok(Ok(_)) => Ok(TerminalTrimOutcome::DeletionPending), - Ok(Err(AppendErrorInternal::StreamDeletionPending(_))) => { + Ok(Err(AppendErrorInternal::StreamDeletionPending { .. })) => { Ok(TerminalTrimOutcome::DeletionPending) } Ok(Err(AppendErrorInternal::Storage(e))) => Err(DeleteStreamError::Storage(e)), @@ -543,31 +589,12 @@ impl Streamer { Ok(Err(AppendErrorInternal::MaxSeqNum(_))) => { unreachable!("terminal append is plaintext command record") } - Err(_) => Err(DeleteStreamError::StreamerMissingInActionError( - StreamerMissingInActionError, - )), + Err(_) => Err(RequestDroppedError.into()), }; let _ = reply_tx.send(result); }); } - fn doe_deadline_maybe(&mut self) -> Option { - let retention_age = self.config.retention_policy.age()?; - let min_age = self.config.delete_on_empty.min_age()?; - let now = Instant::now(); - if self - .last_doe_deadline_at - .is_none_or(|t| now.duration_since(t) >= DOE_DEADLINE_REFRESH_PERIOD) - { - self.last_doe_deadline_at = Some(now); - let deadline = - kv::timestamp::TimestampSecs::after(doe_arm_delay(retention_age, min_age)); - Some(kv::stream_doe_deadline::Entry { deadline, min_age }) - } else { - None - } - } - fn subscribe_durability(&mut self) { if let Some(inflight_append) = self .inflight_appends @@ -659,14 +686,14 @@ impl Streamer { } Message::DeleteOnEmptyCheckResult { stable_pos_snapshot, - last_write_cutoff, - has_records, + config_seq_snapshot, + records, reply_tx, } => { self.handle_doe_check_result( stable_pos_snapshot, - last_write_cutoff, - has_records, + config_seq_snapshot, + records, reply_tx, ); } @@ -684,8 +711,11 @@ impl Streamer { Message::CheckTail { reply_tx } => { let _ = reply_tx.send(self.stable_pos); } - Message::Reconfigure { config } => { - self.config = config; + Message::Reconfigure { seq, config } => { + if seq > self.config_seq { + self.config = config; + self.config_seq = seq; + } } Message::DurabilityStatus(status) => { match status { @@ -706,7 +736,12 @@ impl Streamer { } } _ = dormancy.as_mut() => { - if self.lease_state.close_if_idle() { + // Cancelled requests can still have writes become durable. Keep + // their assigned positions until a new streamer can recover them. + if self.db_writes_pending.is_empty() + && self.inflight_appends.is_empty() + && self.lease_state.close_if_idle() + { break; } } @@ -728,8 +763,8 @@ enum Message { }, DeleteOnEmptyCheckResult { stable_pos_snapshot: StreamPosition, - last_write_cutoff: kv::timestamp::TimestampSecs, - has_records: Result, + config_seq_snapshot: u64, + records: Result, reply_tx: oneshot::Sender>, }, Follow { @@ -742,6 +777,7 @@ enum Message { reply_tx: oneshot::Sender, }, Reconfigure { + seq: u64, config: StreamConfig, }, DurabilityStatus(Result), @@ -750,14 +786,19 @@ enum Message { pub(super) enum TerminalTrimCondition { Always, DeleteOnEmpty { - last_write_cutoff: kv::timestamp::TimestampSecs, + expected_stream_creation_seq: u64, + expected_config_seq: u64, }, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum TerminalTrimOutcome { + /// Deletion is durably pending. DeletionPending, - Ineligible, + RetryAt(TimestampSecs), + Parked, + /// DOE is disabled or this request belongs to an earlier incarnation. + Obsolete, } #[derive(Debug, Clone)] @@ -837,8 +878,10 @@ impl StreamerClient { }) } - pub(super) fn advise_reconfig(&self, config: StreamConfig) -> bool { - self.msg_tx.send(Message::Reconfigure { config }).is_ok() + pub(super) fn advise_reconfig(&self, seq: u64, config: StreamConfig) -> bool { + self.msg_tx + .send(Message::Reconfigure { seq, config }) + .is_ok() } async fn terminal_trim( @@ -854,9 +897,7 @@ impl StreamerClient { .map_err(|_| { DeleteStreamError::StreamerMissingInActionError(StreamerMissingInActionError) })?; - reply_rx.await.map_err(|_| { - DeleteStreamError::StreamerMissingInActionError(StreamerMissingInActionError) - })? + reply_rx.await.map_err(|_| RequestDroppedError)? } } @@ -945,7 +986,17 @@ pub fn next_pos(records: &[Metered]) -> StreamPosition { } } -async fn stream_has_records(db: &slatedb::Db, stream_id: StreamId) -> Result { +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum RecordPresence { + Empty, + ExpiresAt(TimestampSecs), + Unbounded, +} + +async fn stream_record_presence( + db: &slatedb::Db, + stream_id: StreamId, +) -> Result { let prefix = kv::stream_record_timestamp::ser_key_prefix(stream_id); let scan_opts = ScanOptions::default().with_order(IterationOrder::Descending); let mut it = db.scan_prefix_with_options(prefix, .., &scan_opts).await?; @@ -954,11 +1005,20 @@ async fn stream_has_records(db: &slatedb::Db, stream_id: StreamId) -> Result now_millis) { - return Ok(true); + match kv.expire_ts { + None => return Ok(RecordPresence::Unbounded), + Some(expire_ts) if expire_ts > now_millis => { + // One live record bounds when the stream can become empty. + // Use its stored TTL, since retention changes are not retroactive. + // Round up so the check does not run just before expiration. + return Ok(RecordPresence::ExpiresAt(TimestampSecs::from_millis( + expire_ts.saturating_add(999), + ))); + } + _ => (), } } - Ok(false) + Ok(RecordPresence::Empty) } fn sequenced_records( @@ -1013,56 +1073,46 @@ async fn db_submit_append( records: Vec>, DbSubmitAppendOptions { retention, - doe_deadline, fencing_token, trim_point, }: DbSubmitAppendOptions, ) -> Result { let ttl = match retention { - RetentionPolicy::Age(age) => Ttl::ExpireAfter(age.as_millis() as u64), + RetentionPolicy::Age(age) => Ttl::ExpireAfterMillis(age.as_millis() as u64), RetentionPolicy::Infinite() => Ttl::NoExpiry, }; let ttl_put_opts = PutOptions { ttl }; let mut wb = WriteBatch::new(); for (position, record) in records.iter().map(|msr| msr.parts()) { - wb.put_with_options( + wb.put_bytes_with_options( kv::stream_record_data::ser_key(stream_id, position), kv::stream_record_data::ser_value(record), &ttl_put_opts, ); - wb.put_with_options( + wb.put_bytes_with_options( kv::stream_record_timestamp::ser_key(stream_id, position), kv::stream_record_timestamp::ser_value(), &ttl_put_opts, ); } if let Some(fencing_token) = fencing_token { - wb.put( + wb.put_bytes( kv::stream_fencing_token::ser_key(stream_id), kv::stream_fencing_token::ser_value(&fencing_token), ); } if let Some(trim_point) = trim_point.and_then(|tp| NonZeroSeqNum::new(tp.end)) { - wb.put( + wb.put_bytes( kv::stream_trim_point::ser_key(stream_id), kv::stream_trim_point::ser_value(..trim_point), ); } - if let Some(doe_deadline) = doe_deadline { - wb.put( - kv::stream_doe_deadline::ser_key(doe_deadline.deadline, stream_id), - kv::stream_doe_deadline::ser_value(doe_deadline.min_age), - ); - } - wb.put( + wb.put_bytes( kv::stream_tail_position::ser_key(stream_id), kv::stream_tail_position::ser_value(next_pos(&records)), ); - let write_opts = WriteOptions { - await_durable: false, - ..Default::default() - }; - let write_handle = db.write_with_options(wb, &write_opts).await?; + // The durability notifier tracks this sequence and acknowledges the append after flush. + let write_handle = db.write(wb).await?; Ok(InFlightAppend { db_seq: write_handle.seqnum(), records, @@ -1409,9 +1459,27 @@ mod tests { } } + async fn make_pending_append_durable(streamer: &mut Streamer) { + let submitted = streamer + .db_writes_pending + .pop_front() + .unwrap() + .await + .unwrap(); + let seq = submitted.db_seq; + streamer.inflight_appends.push_back(submitted); + streamer.db.flush().await.unwrap(); + streamer.on_db_durable_seq_advanced(seq); + } + async fn test_streamer() -> Streamer { + test_streamer_with_settings(Default::default()).await + } + + async fn test_streamer_with_settings(settings: slatedb::config::Settings) -> Streamer { let object_store = Arc::new(InMemory::new()); let db = slatedb::Db::builder("/test", object_store) + .with_settings(settings) .build() .await .expect("db"); @@ -1421,9 +1489,11 @@ mod tests { Streamer { db: db.clone(), stream_id: [3u8; StreamId::LEN].into(), + stream_creation_seq: 0, msg_tx, config: StreamConfig::default(), - last_tail_write_timestamp: kv::timestamp::TimestampSecs::ZERO, + config_seq: 0, + last_tail_write_timestamp: TimestampSecs::ZERO, fencing_token: CommandState { state: FencingToken::default(), applied_point: ..SeqNum::MIN, @@ -1432,7 +1502,6 @@ mod tests { state: ..SeqNum::MIN, applied_point: ..SeqNum::MIN, }, - last_doe_deadline_at: None, db_writes_pending: VecDeque::new(), db_durability_subscription: 0, inflight_appends: VecDeque::new(), @@ -1445,6 +1514,62 @@ mod tests { } } + #[tokio::test] + async fn stale_config_notifications_cannot_restore_old_retention() { + let mut streamer = test_streamer().await; + let db = streamer.db.clone(); + let stream_id = streamer.stream_id; + let (msg_tx, msg_rx) = mpsc::unbounded_channel(); + streamer.msg_tx = msg_tx.clone(); + streamer.config.retention_policy = RetentionPolicy::Age(Duration::from_secs(1)); + let task = tokio::spawn(streamer.run(msg_rx)); + msg_tx + .send(Message::Reconfigure { + seq: 20, + config: StreamConfig { + retention_policy: RetentionPolicy::Infinite(), + ..Default::default() + }, + }) + .unwrap(); + let old = StreamConfig { + retention_policy: RetentionPolicy::Age(Duration::from_secs(1)), + ..Default::default() + }; + msg_tx + .send(Message::Reconfigure { + seq: 10, + config: old, + }) + .unwrap(); + let (reply_tx, reply_rx) = oneshot::channel(); + msg_tx + .send(Message::Append { + input: append_input(b"must not expire"), + session: None, + reply_tx, + append_type: AppendType::Regular, + }) + .unwrap(); + let ack = tokio::time::timeout(Duration::from_secs(5), reply_rx) + .await + .unwrap() + .unwrap() + .unwrap(); + let entry = db + .get_key_value(kv::stream_record_data::ser_key(stream_id, ack.start)) + .await + .unwrap() + .unwrap(); + assert!( + entry.expire_ts.is_none(), + "late config notification restored stale retention" + ); + task.abort(); + assert!(task.await.unwrap_err().is_cancelled()); + db.close().await.unwrap(); + } + #[test] fn lease_state_closes_when_idle_and_rejects_new_leases() { let (streamer_lease_state, client_lease_state) = StreamerLeaseState::new(); @@ -1489,60 +1614,304 @@ mod tests { } #[tokio::test] - async fn append_during_terminal_trim_returns_stream_deletion_pending() { - let mut streamer = test_streamer().await; - streamer.trim_point = CommandState { - state: ..SeqNum::MAX, - applied_point: ..1, - }; - let (msg_tx, msg_rx) = mpsc::unbounded_channel(); - let run_handle = tokio::spawn(streamer.run(msg_rx)); + async fn terminal_trim_and_rejections_wait_for_durability() { + let mut streamer = test_streamer_with_settings(slatedb::config::Settings { + flush_interval: None, + ..Default::default() + }) + .await; + let mut replies = Vec::new(); + for _ in 0..2 { + let (tx, rx) = oneshot::channel(); + streamer.handle_terminal_trim(TerminalTrimCondition::Always, tx); + replies.push(rx); + } + // An empty-stream check can also finish after another deletion request starts. + let (tx, rx) = oneshot::channel(); + streamer.handle_doe_check_result( + StreamPosition::MIN, + streamer.config_seq, + Ok(RecordPresence::Empty), + tx, + ); + replies.push(rx); - let (reply_tx, reply_rx) = oneshot::channel(); - msg_tx - .send(Message::Append { - input: append_input(b"late"), - session: None, - reply_tx, - append_type: AppendType::Regular, - }) - .expect("streamer should accept append message"); + let (tx, mut append_reply) = oneshot::channel(); + streamer.handle_append(append_input(b"late"), None, tx, AppendType::Regular); + assert_eq!(streamer.db_writes_pending.len(), 1); + tokio::task::yield_now().await; + assert!(matches!( + append_reply.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + )); + for reply in &mut replies { + assert!( + matches!(reply.try_recv(), Err(oneshot::error::TryRecvError::Empty)), + "stream deletion must wait even when the original trim has not been submitted" + ); + } - let err = reply_rx + let submitted = streamer + .db_writes_pending + .pop_front() + .unwrap() .await - .expect("streamer should reply") - .expect_err("append should be rejected"); - let AppendErrorInternal::StreamDeletionPending(_) = err else { - panic!("expected stream deletion pending"); - }; + .unwrap(); + let db_seq = submitted.db_seq; + streamer.inflight_appends.push_back(submitted); + tokio::task::yield_now().await; + assert!(matches!( + append_reply.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + )); + for reply in &mut replies { + assert!( + matches!(reply.try_recv(), Err(oneshot::error::TryRecvError::Empty)), + "a committed but unflushed trim must not acknowledge stream deletion" + ); + } + streamer.db.flush().await.unwrap(); + streamer.on_db_durable_seq_advanced(db_seq); + assert!(matches!( + append_reply.await.unwrap(), + Err(AppendErrorInternal::StreamDeletionPending { .. }) + )); + for reply in replies { + assert_eq!( + reply.await.unwrap().unwrap(), + TerminalTrimOutcome::DeletionPending + ); + } + streamer.db.close().await.unwrap(); + } - run_handle.abort(); + #[rstest::rstest] + #[case::pending_before_scan(false, false)] + #[case::pending_after_scan(true, false)] + #[case::durable_after_scan(true, true)] + #[tokio::test] + async fn delete_on_empty_uses_nonempty_observation_despite_appends( + #[case] append_after_scan: bool, + #[case] durable: bool, + #[values(false, true)] infinite: bool, + ) { + let mut streamer = test_streamer().await; + streamer.config.delete_on_empty.min_age = Duration::from_secs(60); + streamer.config.retention_policy = if infinite { + RetentionPolicy::Infinite() + } else { + RetentionPolicy::Age(Duration::from_secs(3600)) + }; + let (seed_tx, seed_rx) = oneshot::channel(); + streamer.handle_append( + append_input(b"observed record"), + None, + seed_tx, + AppendType::Regular, + ); + make_pending_append_durable(&mut streamer).await; + seed_rx.await.unwrap().unwrap(); + + let (msg_tx, mut msg_rx) = mpsc::unbounded_channel(); + streamer.msg_tx = msg_tx; + let (append_tx, append_rx) = oneshot::channel(); + // Keep the sender until the selected point in the asynchronous check. + let mut append_tx = Some(append_tx); + if !append_after_scan { + streamer.handle_append( + append_input(b"pending before scan"), + None, + append_tx.take().unwrap(), + AppendType::Regular, + ); + } + let (reply_tx, reply_rx) = oneshot::channel(); + streamer.handle_terminal_trim( + TerminalTrimCondition::DeleteOnEmpty { + expected_stream_creation_seq: streamer.stream_creation_seq, + expected_config_seq: streamer.config_seq, + }, + reply_tx, + ); + let Message::DeleteOnEmptyCheckResult { + stable_pos_snapshot, + config_seq_snapshot, + records, + reply_tx, + } = msg_rx.recv().await.unwrap() + else { + panic!("expected record check even with a pending append"); + }; + if append_after_scan { + streamer.handle_append( + append_input(b"arrived after scan"), + None, + append_tx.take().unwrap(), + AppendType::Regular, + ); + } + if durable { + make_pending_append_durable(&mut streamer).await; + append_rx.await.unwrap().unwrap(); + } + streamer.handle_doe_check_result( + stable_pos_snapshot, + config_seq_snapshot, + records, + reply_tx, + ); + assert_eq!(streamer.db_writes_pending.len(), usize::from(!durable)); + assert_ne!(streamer.trim_point.state.end, SeqNum::MAX); + let outcome = reply_rx.await.unwrap().unwrap(); + if infinite { + assert_eq!(outcome, TerminalTrimOutcome::Parked); + } else { + assert!(matches!( + outcome, + TerminalTrimOutcome::RetryAt(at) + if at > TimestampSecs::after(Duration::from_secs(3500)) + )); + } + streamer.db.close().await.unwrap(); + } + + #[rstest::rstest] + #[case::pending_before_scan(false, false)] + #[case::pending_after_scan(true, false)] + #[case::durable_after_scan(true, true)] + #[tokio::test] + async fn delete_on_empty_rechecks_appends_after_empty_scan( + #[case] append_after_scan: bool, + #[case] durable: bool, + ) { + let mut streamer = test_streamer().await; + streamer.config.delete_on_empty.min_age = Duration::from_secs(1); + let (msg_tx, mut msg_rx) = mpsc::unbounded_channel(); + streamer.msg_tx = msg_tx; + let (append_tx, append_rx) = oneshot::channel(); + let mut append_tx = Some(append_tx); + if !append_after_scan { + streamer.handle_append( + append_input(b"pending before scan"), + None, + append_tx.take().unwrap(), + AppendType::Regular, + ); + } + let (reply_tx, reply_rx) = oneshot::channel(); + streamer.handle_terminal_trim( + TerminalTrimCondition::DeleteOnEmpty { + expected_stream_creation_seq: 0, + expected_config_seq: 0, + }, + reply_tx, + ); + let Message::DeleteOnEmptyCheckResult { + stable_pos_snapshot, + config_seq_snapshot, + records, + reply_tx, + } = msg_rx.recv().await.unwrap() + else { + panic!("expected empty-stream check result"); + }; + assert_eq!(records.as_ref().unwrap(), &RecordPresence::Empty); + if append_after_scan { + streamer.handle_append( + append_input(b"arrived during scan"), + None, + append_tx.take().unwrap(), + AppendType::Regular, + ); + } + if durable { + make_pending_append_durable(&mut streamer).await; + append_rx.await.unwrap().unwrap(); + } + // Let the minimum age elapse so it cannot mask a missing tail check. + tokio::time::sleep(Duration::from_millis(1100)).await; + streamer.handle_doe_check_result( + stable_pos_snapshot, + config_seq_snapshot, + records, + reply_tx, + ); + assert_eq!(streamer.db_writes_pending.len(), usize::from(!durable)); + assert_ne!(streamer.trim_point.state.end, SeqNum::MAX); + assert!(matches!( + reply_rx.await.unwrap().unwrap(), + TerminalTrimOutcome::RetryAt(_) + )); + streamer.db.close().await.unwrap(); + } + + #[rstest::rstest] + #[case::actor_behind(0, 1)] + #[case::worker_behind(1, 0)] + #[tokio::test] + async fn delete_on_empty_bounds_retry_for_stale_config( + #[case] actor_seq: u64, + #[case] worker_seq: u64, + ) { + let mut streamer = test_streamer().await; + streamer.config.delete_on_empty.min_age = Duration::from_secs(365 * 24 * 3600); + streamer.last_tail_write_timestamp = TimestampSecs::now(); + streamer.config_seq = actor_seq; + let earliest_retry = TimestampSecs::after(doe::RETRY_INTERVAL); + let (reply_tx, reply_rx) = oneshot::channel(); + streamer.handle_terminal_trim( + TerminalTrimCondition::DeleteOnEmpty { + expected_stream_creation_seq: 0, + expected_config_seq: worker_seq, + }, + reply_tx, + ); + let TerminalTrimOutcome::RetryAt(at) = reply_rx.await.unwrap().unwrap() else { + panic!("expected a bounded retry for mismatched configuration"); + }; + assert!(at >= earliest_retry && at <= TimestampSecs::after(doe::RETRY_INTERVAL)); + assert!(streamer.db_writes_pending.is_empty()); + streamer.db.close().await.unwrap(); } #[tokio::test] - async fn delete_on_empty_terminal_trim_skips_pending_append() { + async fn delete_on_empty_bounds_retry_when_config_changes_during_scan() { let mut streamer = test_streamer().await; - let (append_tx, mut append_rx) = oneshot::channel(); - streamer.handle_append(append_input(b"live"), None, append_tx, AppendType::Regular); - assert_eq!(streamer.db_writes_pending.len(), 1); - assert!(matches!( - append_rx.try_recv(), - Err(tokio::sync::oneshot::error::TryRecvError::Empty) - )); - - let (trim_tx, trim_rx) = oneshot::channel(); + streamer.config.delete_on_empty.min_age = Duration::from_secs(365 * 24 * 3600); + streamer.last_tail_write_timestamp = TimestampSecs::now(); + let (msg_tx, mut msg_rx) = mpsc::unbounded_channel(); + streamer.msg_tx = msg_tx; + let (reply_tx, reply_rx) = oneshot::channel(); streamer.handle_terminal_trim( TerminalTrimCondition::DeleteOnEmpty { - last_write_cutoff: kv::timestamp::TimestampSecs::MAX, + expected_stream_creation_seq: 0, + expected_config_seq: 0, }, - trim_tx, + reply_tx, ); - - assert_eq!( - trim_rx.await.expect("terminal trim reply").unwrap(), - TerminalTrimOutcome::Ineligible + let Message::DeleteOnEmptyCheckResult { + stable_pos_snapshot, + config_seq_snapshot, + records, + reply_tx, + } = msg_rx.recv().await.unwrap() + else { + panic!("expected record check"); + }; + streamer.config_seq += 1; + let earliest_retry = TimestampSecs::after(doe::RETRY_INTERVAL); + streamer.handle_doe_check_result( + stable_pos_snapshot, + config_seq_snapshot, + records, + reply_tx, ); - assert_eq!(streamer.db_writes_pending.len(), 1); + let TerminalTrimOutcome::RetryAt(at) = reply_rx.await.unwrap().unwrap() else { + panic!("expected a bounded retry for mismatched configuration"); + }; + assert!(at >= earliest_retry && at <= TimestampSecs::after(doe::RETRY_INTERVAL)); + assert!(streamer.db_writes_pending.is_empty()); + streamer.db.close().await.unwrap(); } #[tokio::test] @@ -1675,4 +2044,50 @@ mod tests { assert_eq!(streamer.stable_pos.seq_num, 4); assert!(streamer.inflight_appends.is_empty()); } + + #[tokio::test(start_paused = true)] + async fn cancelled_append_delays_dormancy_until_writes_are_durable() { + let mut streamer = test_streamer_with_settings(slatedb::config::Settings { + flush_interval: None, + ..Default::default() + }) + .await; + let db = streamer.db.clone(); + let (msg_tx, msg_rx) = mpsc::unbounded_channel(); + streamer.msg_tx = msg_tx; + + let (reply_tx, reply_rx) = oneshot::channel(); + streamer.handle_append( + append_input(b"cancelled"), + None, + reply_tx, + AppendType::Regular, + ); + let task = tokio::spawn(streamer.run(msg_rx)); + tokio::time::timeout(Duration::from_secs(5), async { + while db.snapshot().await.unwrap().seq() == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + drop(reply_rx); + tokio::time::sleep(DORMANT_TIMEOUT + Duration::from_secs(1)).await; + assert_eq!( + db.status().durable_seq, + 0, + "the write must still be unflushed" + ); + assert!( + !task.is_finished(), + "dormancy abandoned an unflushed append" + ); + + db.flush().await.unwrap(); + tokio::time::timeout(DORMANT_TIMEOUT + Duration::from_secs(1), task) + .await + .expect("durable writes should allow normal dormancy") + .unwrap(); + db.close().await.unwrap(); + } } diff --git a/lite/src/backend/streams.rs b/lite/src/backend/streams.rs index b59d8032..b74ca6f3 100644 --- a/lite/src/backend/streams.rs +++ b/lite/src/backend/streams.rs @@ -1,22 +1,23 @@ use s2_common::{ basin::BasinName, - config::{OptionalStreamConfig, StreamConfig, StreamReconfiguration}, - record::StreamPosition, + config::{DeleteOnEmptyConfig, OptionalStreamConfig, StreamConfig, StreamReconfiguration}, + record::{NonZeroSeqNum, StreamPosition}, resources::{Page, ProvisionMode, ProvisionResult, RequestToken}, stream::{ListStreamsRequest, StreamInfo, StreamName}, }; use s2_storage::bash::Bash; use slatedb::{ - IsolationLevel, + DbTransaction, IsolationLevel, config::{DurabilityLevel, ScanOptions}, }; use time::OffsetDateTime; use tracing::instrument; use super::{ - Backend, - store::db_txn_get, - streamer::{TerminalTrimCondition, TerminalTrimOutcome, doe_arm_delay}, + Backend, doe, resolve_stream_config, + store::{db_txn_commit_durable, db_txn_get, db_txn_get_with}, + streamer::{TerminalTrimCondition, TerminalTrimOutcome}, + timestamp::TimestampSecs, }; use crate::{ backend::{ @@ -106,22 +107,26 @@ impl Backend { // Existence is decided from a Memory-level read; capture the row's // commit seq so exists-outcomes can await durability (see fn doc). - let existing_entry = txn.get_key_value(&stream_meta_key).await?; - let existing_seq = existing_entry.as_ref().map(|kv| kv.seq); - let existing_meta = existing_entry - .map(|kv| kv::stream_meta::deser_value(kv.value)) - .transpose() - .map_err(StorageError::from)?; - if let Some(existing_meta) = &existing_meta - && existing_meta.deleted_at.is_some() + let (existing_meta, existing_seq) = db_txn_get_with(&txn, &stream_meta_key, |entry| { + Ok((kv::stream_meta::deser_value(entry.value)?, entry.seq)) + }) + .await? + .unzip(); + if existing_meta + .as_ref() + .is_some_and(|meta| meta.deleted_at.is_some()) + || has_terminal_trim(&txn, StreamId::new(&basin, &stream)).await? { return Err(ProvisionStreamError::StreamDeletionPending( StreamDeletionPendingError, )); } + let prior_doe = existing_meta + .as_ref() + .map(|meta| meta.config.delete_on_empty); let basin_defaults = basin_meta.config.default_stream_config; - let (outcome, prior_doe_min_age) = match (existing_meta, mode) { + let outcome = match (existing_meta, mode) { (Some(existing), ProvisionMode::CreateOnly { request_token }) => { let new_creation_idempotency_key = request_token .as_ref() @@ -144,7 +149,7 @@ impl Backend { return result; } (Some(existing), ProvisionMode::Ensure) => { - let desired_config = config.merge(basin_defaults); + let desired_config = resolve_stream_config(config, basin_defaults); let config_unchanged = existing.config == desired_config; let meta = kv::stream_meta::StreamMeta { config: desired_config, @@ -153,40 +158,33 @@ impl Backend { deleted_at: None, creation_idempotency_key: existing.creation_idempotency_key, }; - ( - if config_unchanged { - ProvisionResult::Noop(meta) - } else { - ProvisionResult::Updated(meta) - }, - existing.config.delete_on_empty.min_age(), - ) + if config_unchanged { + ProvisionResult::Noop(meta) + } else { + ProvisionResult::Updated(meta) + } } (None, ProvisionMode::CreateOnly { request_token }) => { let new_creation_idempotency_key = request_token .as_ref() .map(|req_token| creation_idempotency_key(req_token, &config)); - ( - ProvisionResult::Created(kv::stream_meta::StreamMeta { - config: config.merge(basin_defaults), - cipher: basin_meta.config.stream_cipher, - created_at: OffsetDateTime::now_utc(), - deleted_at: None, - creation_idempotency_key: new_creation_idempotency_key, - }), - None, - ) - } - (None, ProvisionMode::Ensure) => ( ProvisionResult::Created(kv::stream_meta::StreamMeta { - config: config.merge(basin_defaults), + config: resolve_stream_config(config, basin_defaults), + cipher: basin_meta.config.stream_cipher, + created_at: OffsetDateTime::now_utc(), + deleted_at: None, + creation_idempotency_key: new_creation_idempotency_key, + }) + } + (None, ProvisionMode::Ensure) => { + ProvisionResult::Created(kv::stream_meta::StreamMeta { + config: resolve_stream_config(config, basin_defaults), cipher: basin_meta.config.stream_cipher, created_at: OffsetDateTime::now_utc(), deleted_at: None, creation_idempotency_key: None, - }), - None, - ), + }) + } }; if matches!(&outcome, ProvisionResult::Noop(_)) { @@ -211,28 +209,14 @@ impl Backend { )?; } - if let Some(min_age) = meta.config.delete_on_empty.min_age() - && (matches!(&outcome, ProvisionResult::Created(_)) || prior_doe_min_age.is_none()) - { - txn.put( - kv::stream_doe_deadline::ser_key( - kv::timestamp::TimestampSecs::after(doe_arm_delay( - meta.config.retention_policy.age().unwrap_or_default(), - min_age, - )), - stream_id, - ), - kv::stream_doe_deadline::ser_value(min_age), - )?; - } - - txn.commit().await?; - } - - if let ProvisionResult::Updated(meta) = &outcome - && let Some(client) = self.streamer_client_if_active(&basin, &stream) - { - client.advise_reconfig(meta.config.clone()); + self.commit_stream_config( + txn, + basin.clone(), + stream.clone(), + meta.config.clone(), + prior_doe, + ) + .await?; } Ok(outcome.map(|meta| StreamInfo { @@ -243,17 +227,6 @@ impl Backend { })) } - pub(super) async fn stream_id_mapping( - &self, - stream_id: StreamId, - ) -> Result, StorageError> { - self.db_get( - kv::stream_id_mapping::ser_key(stream_id), - kv::stream_id_mapping::deser_value, - ) - .await - } - pub async fn get_stream_config( &self, basin: BasinName, @@ -305,43 +278,62 @@ impl Backend { stream: stream.clone(), })?; - if meta.deleted_at.is_some() { + if meta.deleted_at.is_some() + || has_terminal_trim(&txn, StreamId::new(&basin, &stream)).await? + { return Err(StreamDeletionPendingError.into()); } - let prior_doe_min_age = meta.config.delete_on_empty.min_age(); + let prior_doe = meta.config.delete_on_empty; - meta.config = OptionalStreamConfig::from(meta.config) - .reconfigure(reconfig) - .merge(basin_meta.config.default_stream_config); + meta.config = resolve_stream_config( + OptionalStreamConfig::from(meta.config).reconfigure(reconfig), + basin_meta.config.default_stream_config, + ); txn.put(&meta_key, kv::stream_meta::ser_value(&meta))?; - let stream_id = StreamId::new(&basin, &stream); - if let Some(min_age) = meta.config.delete_on_empty.min_age() - && prior_doe_min_age.is_none() - { - txn.put( - kv::stream_doe_deadline::ser_key( - kv::timestamp::TimestampSecs::after(doe_arm_delay( - meta.config.retention_policy.age().unwrap_or_default(), - min_age, - )), - stream_id, - ), - kv::stream_doe_deadline::ser_value(min_age), - )?; - } - - txn.commit().await?; - - if let Some(client) = self.streamer_client_if_active(&basin, &stream) { - client.advise_reconfig(meta.config.clone()); - } + self.commit_stream_config(txn, basin, stream, meta.config.clone(), Some(prior_doe)) + .await?; Ok(meta.config) } + async fn commit_stream_config( + &self, + txn: DbTransaction, + basin: BasinName, + stream: StreamName, + config: StreamConfig, + prior_doe: Option, + ) -> Result<(), StorageError> { + let stream_id = StreamId::new(&basin, &stream); + match (prior_doe, config.delete_on_empty.min_age()) { + (None, Some(min_age)) => { + doe::schedule(&txn, stream_id, TimestampSecs::after(min_age)).await?; + } + (Some(prior), Some(min_age)) if prior.min_age != min_age => { + doe::schedule(&txn, stream_id, TimestampSecs::now()).await?; + } + (Some(prior), None) if prior.min_age().is_some() => { + doe::clear(&txn, stream_id).await?; + } + _ => (), + } + + let backend = self.clone(); + // Once a commit starts, cancellation must not discard its notification. + tokio::spawn(async move { + let seq = db_txn_commit_durable(txn) + .await? + .expect("stream metadata was written"); + backend.advise_stream_config(&basin, &stream, seq, config); + Ok::<_, StorageError>(()) + }) + .await + .expect("stream config commit task panicked") + } + #[instrument(ret, err, skip(self))] pub async fn delete_stream( &self, @@ -350,6 +342,7 @@ impl Backend { ) -> Result<(), DeleteStreamError> { self.delete_stream_with_condition(basin, stream, TerminalTrimCondition::Always) .await + .map(|_| ()) } pub(super) async fn delete_stream_with_condition( @@ -357,7 +350,7 @@ impl Backend { basin: BasinName, stream: StreamName, condition: TerminalTrimCondition, - ) -> Result<(), DeleteStreamError> { + ) -> Result { let outcome = match self.streamer_client_guarded(&basin, &stream).await { Ok(client) => client.terminal_trim(condition).await?, Err(StreamerError::Storage(e)) => { @@ -368,10 +361,10 @@ impl Backend { } Err(StreamerError::StreamDeletionPending(_)) => TerminalTrimOutcome::DeletionPending, }; - match outcome { - TerminalTrimOutcome::DeletionPending => self.mark_stream_deleted(basin, stream).await, - TerminalTrimOutcome::Ineligible => Ok(()), + if outcome == TerminalTrimOutcome::DeletionPending { + self.mark_stream_deleted(basin, stream).await?; } + Ok(outcome) } async fn mark_stream_deleted( @@ -380,22 +373,49 @@ impl Backend { stream: StreamName, ) -> Result<(), DeleteStreamError> { let txn = self.db.begin(IsolationLevel::SerializableSnapshot).await?; + // A delayed deletion request may resume after the trim worker has deleted the old + // stream and the name has been reused. Only mark metadata when this + // transaction also sees a terminal trim marker. + if !has_terminal_trim(&txn, StreamId::new(&basin, &stream)).await? { + let read_seq = txn.seqnum(); + drop(txn); + // The trim worker may have removed the marker without flushing yet. + // Wait for that removal to become durable before acknowledging deletion. + self.await_durable_seq(read_seq).await?; + return Ok(()); + } let meta_key = kv::stream_meta::ser_key(&basin, &stream); - let mut meta = db_txn_get(&txn, &meta_key, kv::stream_meta::deser_value) - .await? - .ok_or_else(|| StreamNotFoundError { - basin, - stream: stream.clone(), - })?; + let (mut meta, seq) = db_txn_get_with(&txn, &meta_key, |entry| { + Ok((kv::stream_meta::deser_value(entry.value)?, entry.seq)) + }) + .await? + .ok_or_else(|| StreamNotFoundError { + basin, + stream: stream.clone(), + })?; if meta.deleted_at.is_none() { meta.deleted_at = Some(OffsetDateTime::now_utc()); txn.put(&meta_key, kv::stream_meta::ser_value(&meta))?; - txn.commit().await?; + db_txn_commit_durable(txn).await?; + } else { + // The terminal trim is durable, but the metadata marker may not be yet. + drop(txn); + self.await_durable_seq(seq).await?; } Ok(()) } } +async fn has_terminal_trim(txn: &DbTransaction, stream_id: StreamId) -> Result { + Ok(db_txn_get( + txn, + kv::stream_trim_point::ser_key(stream_id), + kv::stream_trim_point::deser_value, + ) + .await? + == Some(..NonZeroSeqNum::MAX)) +} + fn creation_idempotency_key(req_token: &RequestToken, config: &OptionalStreamConfig) -> Bash { Bash::length_prefixed(&[ req_token.as_bytes(), diff --git a/lite/src/backend/test_util.rs b/lite/src/backend/test_util.rs new file mode 100644 index 00000000..3c45f38a --- /dev/null +++ b/lite/src/backend/test_util.rs @@ -0,0 +1,44 @@ +use std::future::Future; + +use slatedb::{Error, WriteHandle}; + +/// Finish a fixture write and assert that remote reads can observe it. +pub(super) trait DbWriteTestExt: + Future> + Sized +{ + async fn assert_durable(self) { + self.await + .expect("fixture write should succeed") + .await_durable() + .await + .expect("fixture write should become durable"); + } +} + +impl>> DbWriteTestExt for F {} + +/// Provision a real stream for lifecycle tests, including its ID mapping and tail. +pub(super) async fn create_stream( + backend: &super::Backend, + config: s2_common::config::OptionalStreamConfig, +) -> (s2_common::basin::BasinName, s2_common::stream::StreamName) { + use s2_common::{config::BasinConfig, resources::ProvisionMode}; + let basin: s2_common::basin::BasinName = "test-basin".parse().unwrap(); + let stream: s2_common::stream::StreamName = "test-stream".parse().unwrap(); + backend + .provision_basin(basin.clone(), BasinConfig::default(), ProvisionMode::Ensure) + .await + .unwrap(); + backend + .provision_stream( + basin.clone(), + stream.clone(), + config, + ProvisionMode::CreateOnly { + request_token: None, + }, + ) + .await + .unwrap(); + (basin, stream) +} diff --git a/lite/src/backend/kv/timestamp.rs b/lite/src/backend/timestamp.rs similarity index 91% rename from lite/src/backend/kv/timestamp.rs rename to lite/src/backend/timestamp.rs index ae368b8e..1455cac7 100644 --- a/lite/src/backend/kv/timestamp.rs +++ b/lite/src/backend/timestamp.rs @@ -44,6 +44,14 @@ impl TimestampSecs { .map(|secs| Self(secs as u32)) } + pub fn saturating_add_duration(self, dur: Duration) -> Self { + Self( + u64::from(self.0) + .saturating_add(dur.as_secs()) + .min(u64::from(u32::MAX)) as u32, + ) + } + fn from_system_time(time: SystemTime) -> Self { match time.duration_since(UNIX_EPOCH) { Ok(duration) => { diff --git a/lite/src/handlers/v1/error.rs b/lite/src/handlers/v1/error.rs index b25e34b0..e589edc2 100644 --- a/lite/src/handlers/v1/error.rs +++ b/lite/src/handlers/v1/error.rs @@ -177,7 +177,7 @@ impl ServiceError { } DeleteStreamError::RequestDroppedError(e) => { // Unavailable error code promised to be side-effect free, - // AppendType::Terminal may have become durable prior to drop. + // The terminal trim marker may have become durable prior to drop. standard(ErrorCode::Other, e.to_string()) } DeleteStreamError::StreamNotFound(e) => { @@ -239,7 +239,7 @@ impl ServiceError { } AppendError::RequestDroppedError(e) => { // Unavailable error code promised to be side-effect free, - // AppendType::Regular may have become durable prior to drop. + // The append may have become durable prior to drop. standard(ErrorCode::Other, e.to_string()) } AppendError::BasinNotFound(e) => standard(ErrorCode::BasinNotFound, e.to_string()), diff --git a/lite/src/handlers/v1/records.rs b/lite/src/handlers/v1/records.rs index 5532e20f..10fe2907 100644 --- a/lite/src/handlers/v1/records.rs +++ b/lite/src/handlers/v1/records.rs @@ -46,11 +46,15 @@ fn validate_read_until(start: ReadStart, end: ReadEnd) -> Result<(), ServiceErro Ok(()) } +/// Adjusts the read bounds to resume after `last_event_id`, and returns the +/// count/bytes already delivered so emitted event ids stay cumulative across +/// reconnects. fn apply_last_event_id( mut start: ReadStart, mut end: v1t::stream::ReadEnd, last_event_id: Option, -) -> (ReadStart, v1t::stream::ReadEnd) { +) -> (ReadStart, v1t::stream::ReadEnd, CountOrBytes) { + let mut delivered = CountOrBytes::ZERO; if let Some(v1t::stream::sse::LastEventId { seq_num, count, @@ -60,8 +64,9 @@ fn apply_last_event_id( start.from = ReadFrom::SeqNum(seq_num.saturating_add(1)); end.count = end.count.map(|c| c.saturating_sub(count)); end.bytes = end.bytes.map(|c| c.saturating_sub(bytes)); + delivered = CountOrBytes { count, bytes }; } - (start, end) + (start, end, delivered) } enum ReadMode { @@ -161,6 +166,7 @@ pub struct ReadArgs { v1t::StreamNamePathSegment, s2_api::data::S2FormatHeader, s2_api::data::S2EncryptionKeyHeader, + s2_api::data::S2StreamConfigHeader, v1t::stream::ReadStart, v1t::stream::ReadEnd, ), @@ -186,12 +192,13 @@ pub async fn read( match request { v1t::stream::ReadRequest::Unary { encryption_key, + create_stream_config_patch, format, response_mime, } => { let (start, end) = prepare_read(start, end, ReadMode::Unary)?; let session = backend - .open_for_read(&basin, &stream, encryption_key) + .open_for_read(&basin, &stream, encryption_key, create_stream_config_patch) .await? .read(start, end) .await?; @@ -209,18 +216,19 @@ pub async fn read( } v1t::stream::ReadRequest::EventStream { encryption_key, + create_stream_config_patch, format, last_event_id, } => { - let (start, end) = apply_last_event_id(start, end, last_event_id); + let (start, end, delivered) = apply_last_event_id(start, end, last_event_id); let (start, end) = prepare_read(start, end, ReadMode::Streaming)?; let session = backend - .open_for_read(&basin, &stream, encryption_key) + .open_for_read(&basin, &stream, encryption_key, create_stream_config_patch) .await? .read(start, end) .await?; let events = async_stream::stream! { - let mut processed = CountOrBytes::ZERO; + let mut processed = delivered; tokio::pin!(session); let mut errored = false; while let Some(output) = session.next().await { @@ -232,8 +240,8 @@ pub async fn read( let Some(last_record) = batch.records.last() else { continue; }; - processed.count += batch.records.len(); - processed.bytes += batch.records.metered_size(); + processed.count = processed.count.saturating_add(batch.records.len()); + processed.bytes = processed.bytes.saturating_add(batch.records.metered_size()); let id = v1t::stream::sse::LastEventId { seq_num: last_record.position().seq_num, count: processed.count, @@ -265,11 +273,12 @@ pub async fn read( } v1t::stream::ReadRequest::S2s { encryption_key, + create_stream_config_patch, response_compression, } => { let (start, end) = prepare_read(start, end, ReadMode::Streaming)?; let s2s_stream = backend - .open_for_read(&basin, &stream, encryption_key) + .open_for_read(&basin, &stream, encryption_key, create_stream_config_patch) .await? .read(start, end) .await? @@ -364,6 +373,7 @@ pub struct AppendArgs { v1t::StreamNamePathSegment, s2_api::data::S2FormatHeader, s2_api::data::S2EncryptionKeyHeader, + s2_api::data::S2StreamConfigHeader, ), servers( (url = super::paths::cloud_endpoints::BASIN, variables( @@ -384,11 +394,12 @@ pub async fn append( match request { v1t::stream::AppendRequest::Unary { encryption_key, + create_stream_config_patch, input, response_mime, } => { let handle = backend - .open_for_append(&basin, &stream, encryption_key) + .open_for_append(&basin, &stream, encryption_key, create_stream_config_patch) .await?; let ack = handle.append(input).await?; match response_mime { @@ -404,11 +415,12 @@ pub async fn append( } v1t::stream::AppendRequest::S2s { encryption_key, + create_stream_config_patch, inputs, response_compression, } => { let handle = backend - .open_for_append(&basin, &stream, encryption_key) + .open_for_append(&basin, &stream, encryption_key, create_stream_config_patch) .await?; let (err_tx, err_rx) = tokio::sync::oneshot::channel(); @@ -469,16 +481,22 @@ mod tests { use bytesize::ByteSize; use futures::TryStreamExt as _; use prost::Message as _; - use s2_api::v1::stream::{ - proto, - s2s::{FrameDecoder, SessionMessage}, + use s2_api::v1::{ + config::STREAM_CONFIG_HEADER, + stream::{ + proto, + s2s::{self, FrameDecoder, SessionMessage}, + sse::LastEventId, + }, }; use s2_common::{ basin::{BASIN_HEADER, BasinName}, - config::{BasinConfig, OptionalStreamConfig}, + config::{ + BasinConfig, DeleteOnEmptyConfig, OptionalStreamConfig, RetentionPolicy, StreamConfig, + }, encryption::{EncryptionAlgorithm, EncryptionKey, S2_ENCRYPTION_KEY_HEADER}, read_extent::{ReadLimit, ReadUntil}, - record::{EnvelopeRecord, Metered, Record}, + record::{EnvelopeRecord, Metered, MeteredSize as _, Record}, resources::ProvisionMode, stream::{ AppendInput, AppendRecord, AppendRecordBatch, AppendRecordParts, ListStreamsRequest, @@ -597,7 +615,12 @@ mod tests { encryption_key: EncryptionKey, ) { backend - .open_for_append(basin, stream, Some(encryption_key)) + .open_for_append( + basin, + stream, + Some(encryption_key), + OptionalStreamConfig::default(), + ) .await .expect("open append handle") .append(append_input(body)) @@ -605,6 +628,95 @@ mod tests { .expect("append encrypted payload"); } + async fn append_payload( + backend: &Backend, + basin: &BasinName, + stream: &StreamName, + body: &'static [u8], + ) { + backend + .open_for_append(basin, stream, None, OptionalStreamConfig::default()) + .await + .expect("open append handle") + .append(append_input(body)) + .await + .expect("append payload"); + } + + struct SseBatches { + seq_nums: Vec, + last_id: Option, + } + + async fn sse_read( + app: &axum::Router, + basin: &BasinName, + stream: &StreamName, + bounds: &str, + last_event_id: Option, + ) -> SseBatches { + let mut request = request_builder( + "GET", + format!("/v1/streams/{stream}/records?seq_num=0&wait=0&{bounds}"), + basin, + ) + .header(header::ACCEPT, "text/event-stream"); + if let Some(id) = last_event_id { + request = request.header("last-event-id", id.to_string()); + } + let response = send(app, request.body(Body::empty()).unwrap()).await; + assert_eq!(response.status(), StatusCode::OK); + let body = + tokio::time::timeout(Duration::from_secs(5), response_bytes(response, "sse body")) + .await + .expect("SSE read should terminate"); + let body = std::str::from_utf8(&body).expect("utf8 sse body"); + + let mut batches = SseBatches { + seq_nums: Vec::new(), + last_id: None, + }; + let mut done = false; + for event in body.split("\n\n").filter(|e| !e.trim().is_empty()) { + let field = |name: &str| { + event + .lines() + .find_map(|line| line.strip_prefix(name)) + .map(str::trim) + }; + match field("event:") { + Some("batch") => { + let id: LastEventId = field("id:") + .expect("batch event id") + .parse() + .expect("parse batch event id"); + let data: serde_json::Value = + serde_json::from_str(field("data:").expect("batch event data")) + .expect("batch event json"); + let records = data["records"].as_array().expect("records array"); + batches.seq_nums.extend( + records + .iter() + .map(|r| r["seq_num"].as_u64().expect("seq_num")), + ); + assert_eq!( + id.seq_num, + *batches.seq_nums.last().expect("non-empty batch") + ); + batches.last_id = Some(id); + } + Some("error") => panic!("unexpected sse error event: {event}"), + Some(other) => panic!("unexpected sse event `{other}`: {event}"), + None => { + assert_eq!(field("data:"), Some("[DONE]"), "unexpected event: {event}"); + done = true; + } + } + } + assert!(done, "SSE read should end with [DONE]"); + batches + } + fn read_uri(stream: &StreamName) -> String { format!("/v1/streams/{stream}/records?seq_num=0&wait=0") } @@ -697,7 +809,12 @@ mod tests { assert_eq!(ack.end.as_ref().map(|pos| pos.seq_num), Some(1)); let records = backend - .open_for_read(&basin, &stream, Some(encryption_key.clone())) + .open_for_read( + &basin, + &stream, + Some(encryption_key.clone()), + OptionalStreamConfig::default(), + ) .await .expect("open read handle") .read( @@ -731,6 +848,240 @@ mod tests { assert_eq!(record.body().as_ref(), b"secret"); } + fn basin_config_with_create_stream_on_append() -> BasinConfig { + BasinConfig { + create_stream_on_append: true, + default_stream_config: OptionalStreamConfig { + storage_class: Some("standard".into()), + ..Default::default() + }, + ..Default::default() + } + } + + fn expected_auto_created_config() -> StreamConfig { + StreamConfig { + storage_class: Some("standard".into()), + retention_policy: RetentionPolicy::Age(Duration::from_secs(3600)), + timestamping: Default::default(), + delete_on_empty: DeleteOnEmptyConfig { + min_age: Duration::from_secs(300), + }, + } + } + + const STREAM_CONFIG_HEADER_VALUE: &str = + r#"{"retention_policy":{"age":3600},"delete_on_empty":{"min_age_secs":300}}"#; + + fn append_record_input(body: &'static [u8]) -> proto::AppendInput { + proto::AppendInput { + records: vec![proto::AppendRecord { + timestamp: None, + headers: vec![], + body: Bytes::from_static(body), + }], + match_seq_num: None, + fencing_token: None, + } + } + + #[tokio::test] + async fn json_append_auto_creates_stream_with_stream_config_header() { + let (app, backend, basin, stream) = setup_app_without_stream( + "append-json-create-config", + basin_config_with_create_stream_on_append(), + ) + .await; + + let body = serde_json::json!({"records": [{"body": "hello"}]}); + let response = send( + &app, + request_builder("POST", format!("/v1/streams/{stream}/records"), &basin) + .header(header::CONTENT_TYPE, "application/json") + .header(STREAM_CONFIG_HEADER.as_str(), STREAM_CONFIG_HEADER_VALUE) + .body(Body::from(body.to_string())) + .unwrap(), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let ack = response_json(response, "append ack body").await; + assert_eq!(ack["end"]["seq_num"], 1); + let config = backend + .get_stream_config(basin, stream) + .await + .expect("get stream config"); + assert_eq!(config, expected_auto_created_config()); + } + + #[tokio::test] + async fn append_with_invalid_stream_config_header_is_rejected_without_creating() { + let (app, backend, basin, stream) = setup_app_without_stream( + "append-create-config-invalid", + basin_config_with_create_stream_on_append(), + ) + .await; + + for (value, expected_message) in [ + ( + r#"{"retention_policy":{"age":0}}"#, + "age must be greater than 0 seconds", + ), + ("not json", "invalid JSON"), + ] { + let body = serde_json::json!({"records": [{"body": "hello"}]}); + let response = send( + &app, + request_builder("POST", format!("/v1/streams/{stream}/records"), &basin) + .header(header::CONTENT_TYPE, "application/json") + .header(STREAM_CONFIG_HEADER.as_str(), value) + .body(Body::from(body.to_string())) + .unwrap(), + ) + .await; + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let info = response_json(response, "append error body").await; + assert_eq!(info["code"], "bad_header"); + let message = info["message"].as_str().expect("error message string"); + assert!( + message.contains("s2-stream-config") && message.contains(expected_message), + "{message}" + ); + } + assert_no_streams(&backend, &basin).await; + } + + #[tokio::test] + async fn s2s_append_session_auto_creates_stream_with_stream_config_header() { + let (app, backend, basin, stream) = setup_app_without_stream( + "append-s2s-create-config", + basin_config_with_create_stream_on_append(), + ) + .await; + + let frame = |body: &'static [u8]| { + SessionMessage::regular(s2s::CompressionAlgorithm::None, &append_record_input(body)) + .expect("encode frame") + .encode() + }; + let mut body = BytesMut::new(); + body.extend_from_slice(&frame(b"first")); + body.extend_from_slice(&frame(b"second")); + + let response = send( + &app, + request_builder("POST", format!("/v1/streams/{stream}/records"), &basin) + .header(header::CONTENT_TYPE, "s2s/proto") + .header(STREAM_CONFIG_HEADER.as_str(), STREAM_CONFIG_HEADER_VALUE) + .body(Body::from(body.freeze())) + .unwrap(), + ) + .await; + + assert_eq!(response.status(), StatusCode::OK); + let body = response_bytes(response, "s2s body").await; + let mut decoder = FrameDecoder; + let mut buf = BytesMut::from(body.as_ref()); + let mut acks = Vec::new(); + while let Some(frame) = decoder.decode(&mut buf).expect("frame decode") { + let SessionMessage::Regular(ack) = frame else { + panic!("expected regular frame"); + }; + acks.push( + ack.try_into_proto::() + .expect("decode append ack"), + ); + } + assert_eq!(acks.len(), 2); + assert_eq!(acks[1].end.as_ref().map(|pos| pos.seq_num), Some(2)); + + let config = backend + .get_stream_config(basin, stream) + .await + .expect("get stream config"); + assert_eq!(config, expected_auto_created_config()); + } + + #[rstest::rstest] + #[case::count("count=0")] + #[case::bytes("bytes=0")] + #[tokio::test] + async fn unary_read_with_zero_limit_returns_empty( + #[case] bound: &str, + #[values( + "timestamp=0", + "seq_num=0", + "tail_offset=0", + "timestamp=1", + "seq_num=1" + )] + start: &str, + #[values(0, 60)] wait: u32, + ) { + let (app, _backend, basin, stream) = setup_app_with_config( + "read-zero-limit", + BasinConfig::default(), + OptionalStreamConfig::default(), + ) + .await; + + let response = tokio::time::timeout( + Duration::from_secs(2), + send( + &app, + request_builder( + "GET", + format!("/v1/streams/{stream}/records?{start}&{bound}&wait={wait}"), + &basin, + ) + .body(Body::empty()) + .unwrap(), + ), + ) + .await + .expect("a zero limit should complete without waiting for records"); + + let status = response.status(); + let body = response_json(response, "read zero-limit response").await; + assert_eq!( + status, + StatusCode::OK, + "{start}&{bound}&wait={wait}: {body}" + ); + assert_eq!(body["records"], serde_json::json!([])); + } + + #[tokio::test] + async fn read_auto_creates_stream_with_stream_config_header() { + let basin_config = BasinConfig { + create_stream_on_append: false, + create_stream_on_read: true, + ..basin_config_with_create_stream_on_append() + }; + let (app, backend, basin, stream) = + setup_app_without_stream("read-create-config", basin_config).await; + + let response = send( + &app, + request_builder( + "GET", + format!("/v1/streams/{stream}/records?seq_num=0"), + &basin, + ) + .header(STREAM_CONFIG_HEADER.as_str(), STREAM_CONFIG_HEADER_VALUE) + .body(Body::empty()) + .unwrap(), + ) + .await; + assert_eq!(response.status(), StatusCode::RANGE_NOT_SATISFIABLE); + let config = backend + .get_stream_config(basin, stream) + .await + .expect("get stream config"); + assert_eq!(config, expected_auto_created_config()); + } + #[tokio::test] async fn invalid_read_bounds_do_not_auto_create_stream() { let basin_config = BasinConfig { @@ -834,6 +1185,103 @@ mod tests { ); } + #[rstest::rstest] + #[case::count(Some(10), None)] + #[case::bytes(None, Some(10))] + #[case::count_first(Some(10), Some(15))] + #[case::bytes_first(Some(15), Some(10))] + #[tokio::test] + async fn sse_resume_emits_cumulative_ids_and_honors_bounds( + #[case] count: Option, + #[case] bytes_in_records: Option, + ) { + let (app, backend, basin, stream) = setup_app_with_config( + "read-sse-resume", + BasinConfig::default(), + OptionalStreamConfig::default(), + ) + .await; + let per_record = append_input(b"payload").records.metered_size(); + let mut bounds = String::new(); + if let Some(count) = count { + bounds.push_str(&format!("count={count}&")); + } + if let Some(records) = bytes_in_records { + bounds.push_str(&format!("bytes={}&", records * per_record)); + } + + // End two connections at the current tail, then resume with exactly + // the same bounds and the actual id emitted by the previous connection. + let mut last_id = None; + let mut delivered = Vec::new(); + for (appended, expected_total) in [(4, 4), (3, 7), (13, 10)] { + for _ in 0..appended { + append_payload(&backend, &basin, &stream, b"payload").await; + } + let resumed = sse_read(&app, &basin, &stream, &bounds, last_id).await; + delivered.extend(resumed.seq_nums); + assert_eq!(delivered, (0..expected_total as u64).collect::>()); + let id = resumed.last_id.expect("last id"); + assert_eq!(id.seq_num, expected_total as u64 - 1); + assert_eq!(id.count, expected_total); + assert_eq!(id.bytes, expected_total * per_record); + last_id = Some(id); + } + + // The same read without reconnects produces the same records and id. + let full = sse_read(&app, &basin, &stream, &bounds, None).await; + assert_eq!(full.seq_nums, delivered); + assert_eq!( + full.last_id.expect("full id").to_string(), + last_id.expect("resumed id").to_string(), + ); + + let exhausted = sse_read(&app, &basin, &stream, &bounds, last_id).await; + assert!(exhausted.seq_nums.is_empty(), "{:?}", exhausted.seq_nums); + assert!(exhausted.last_id.is_none()); + } + + #[rstest::rstest] + #[case::unbounded("", (usize::MAX - 1, usize::MAX - 1), (usize::MAX, usize::MAX), 3)] + #[case::count_bound("count=2", (0, usize::MAX), (2, usize::MAX), 2)] + #[case::bytes_bound("bytes=30", (usize::MAX, 0), (usize::MAX, 30), 2)] + #[tokio::test] + async fn sse_resume_saturates_client_supplied_counters( + #[case] bounds: &str, + #[case] (count, bytes): (usize, usize), + #[case] expected_totals: (usize, usize), + #[case] expected_records: usize, + ) { + let (app, backend, basin, stream) = setup_app_with_config( + "read-sse-saturation", + BasinConfig::default(), + OptionalStreamConfig::default(), + ) + .await; + // Each "payload" record occupies 15 metered bytes. + for _ in 0..4 { + append_payload(&backend, &basin, &stream, b"payload").await; + } + let resumed = sse_read( + &app, + &basin, + &stream, + bounds, + Some(LastEventId { + seq_num: 0, + count, + bytes, + }), + ) + .await; + assert_eq!( + resumed.seq_nums, + (1..=expected_records as u64).collect::>() + ); + let id = resumed.last_id.expect("last id"); + assert_eq!((id.count, id.bytes), expected_totals); + } + #[tokio::test] async fn s2s_read_without_key_header_is_rejected_before_stream_starts() { let encryption_key = aegis_key(0x42); diff --git a/lite/src/server.rs b/lite/src/server.rs index bb4ec354..d78f71f7 100644 --- a/lite/src/server.rs +++ b/lite/src/server.rs @@ -42,15 +42,47 @@ pub struct LiteArgs { /// Name of the S3 bucket to back the database. /// /// If not specified, in-memory storage is used unless --local-root is set. - #[arg(long)] + /// Uses the standard AWS configuration for the endpoint, region and credentials. + #[arg(long, group = "main_store")] pub bucket: Option, /// Root directory to back the database on the local filesystem. /// /// Conflicts with --bucket. - #[arg(long, value_name = "DIR", conflicts_with = "bucket")] + #[arg( + long, + value_name = "DIR", + conflicts_with = "bucket", + group = "main_store" + )] pub local_root: Option, + /// Name of the S3 bucket to back the write-ahead log (WAL). + /// + /// If not specified, the main store is used unless --wal-local-root is set. + /// Uses the same AWS configuration as --bucket, with optional + /// S2LITE_WAL_AWS_* overrides for the endpoint, region and credentials. + /// + /// Requires --bucket or --local-root. Conflicts with --wal-local-root. + #[arg( + long, + env = "S2LITE_WAL_BUCKET", + requires = "main_store", + conflicts_with = "wal_local_root" + )] + pub wal_bucket: Option, + + /// Root directory to back the write-ahead log (WAL) on the local filesystem. + /// + /// Requires --bucket or --local-root. Conflicts with --wal-bucket. + #[arg( + long, + env = "S2LITE_WAL_LOCAL_ROOT", + value_name = "DIR", + requires = "main_store" + )] + pub wal_local_root: Option, + /// Base path on object storage. #[arg(long, default_value = "")] pub path: String, @@ -173,22 +205,44 @@ pub async fn run(args: LiteArgs) -> eyre::Result<()> { StoreType::InMemory }; - let object_store = init_object_store(&store_type).await?; + let object_store = init_object_store(&store_type, None).await?; + let wal_store_type = if let Some(bucket) = args.wal_bucket { + Some(StoreType::S3Bucket(bucket)) + } else { + args.wal_local_root.map(StoreType::LocalFileSystem) + }; + let wal_object_store = match &wal_store_type { + Some(wal_store_type) => { + let s3_overrides = match wal_store_type { + StoreType::S3Bucket(_) => Some(WalS3Overrides::from_env()?), + StoreType::LocalFileSystem(_) | StoreType::InMemory => None, + }; + Some(init_object_store(wal_store_type, s3_overrides).await?) + } + None => None, + }; let db_settings = slatedb::Settings::from_env_with_default( "SL8_", slatedb::Settings { - flush_interval: Some(store_type.default_flush_interval()), + flush_interval: Some( + wal_store_type + .as_ref() + .unwrap_or(&store_type) + .default_flush_interval(), + ), ..Default::default() }, )?; let manifest_poll_interval = db_settings.manifest_poll_interval; - let db = slatedb::Db::builder(args.path, object_store) - .with_settings(db_settings) - .build() - .await?; + let mut builder = slatedb::Db::builder(args.path, object_store).with_settings(db_settings); + if let Some(wal_object_store) = wal_object_store { + info!(store = ?wal_store_type, "using dedicated WAL object store"); + builder = builder.with_wal_object_store(wal_object_store); + } + let db = builder.build().await?; info!( ?manifest_poll_interval, @@ -295,58 +349,16 @@ pub async fn run(args: LiteArgs) -> eyre::Result<()> { async fn init_object_store( store_type: &StoreType, + s3_overrides: Option, ) -> eyre::Result> { - Ok(match store_type { + let store: Arc = match store_type { StoreType::S3Bucket(bucket) => { info!(bucket, "using s3 object store"); - let mut builder = - object_store::aws::AmazonS3Builder::from_env().with_bucket_name(bucket); - - if let Some(endpoint) = - std::env::var_os("AWS_ENDPOINT_URL_S3").and_then(|s| s.into_string().ok()) - { - if endpoint.starts_with("http://") { - builder = builder.with_allow_http(true); - } - builder = builder.with_endpoint(endpoint); + let mut builder = s3_builder().await; + if let Some(overrides) = s3_overrides { + builder = overrides.apply(builder); } - - match ( - std::env::var_os("AWS_ACCESS_KEY_ID").and_then(|s| s.into_string().ok()), - std::env::var_os("AWS_SECRET_ACCESS_KEY").and_then(|s| s.into_string().ok()), - ) { - (Some(key_id), Some(secret_key)) => { - info!(key_id, "using static credentials from env vars"); - - let token = - std::env::var_os("AWS_SESSION_TOKEN").and_then(|s| s.into_string().ok()); - builder = builder.with_credentials(Arc::new( - object_store::StaticCredentialProvider::new( - object_store::aws::AwsCredential { - key_id, - secret_key, - token, - }, - ), - )); - } - _ => { - let aws_config = - aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; - if let Some(region) = aws_config.region() { - info!(region = region.as_ref()); - builder = builder.with_region(region.to_string()); - } - if let Some(credentials_provider) = aws_config.credentials_provider() { - info!("using aws-config credentials provider"); - builder = builder.with_credentials(Arc::new(S3CredentialProvider { - aws: credentials_provider.clone(), - cache: tokio::sync::Mutex::new(None), - })); - } - } - } - Arc::new(builder.build()?) as Arc + Arc::new(builder.with_bucket_name(bucket).build()?) } StoreType::LocalFileSystem(local_root) => { std::fs::create_dir_all(local_root)?; @@ -364,7 +376,127 @@ async fn init_object_store( info!("using in-memory object store"); Arc::new(object_store::memory::InMemory::new()) } - }) + }; + Ok(store) +} + +// Both buckets start with the same AWS configuration and credential chain. +async fn s3_builder() -> object_store::aws::AmazonS3Builder { + let mut builder = object_store::aws::AmazonS3Builder::from_env(); + + if let Some(endpoint) = + std::env::var_os("AWS_ENDPOINT_URL_S3").and_then(|s| s.into_string().ok()) + { + if endpoint.starts_with("http://") { + builder = builder.with_allow_http(true); + } + builder = builder.with_endpoint(endpoint); + } + + match ( + std::env::var_os("AWS_ACCESS_KEY_ID").and_then(|s| s.into_string().ok()), + std::env::var_os("AWS_SECRET_ACCESS_KEY").and_then(|s| s.into_string().ok()), + ) { + (Some(key_id), Some(secret_key)) => { + info!(key_id, "using static credentials from env vars"); + + let aws_config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; + if let Some(region) = aws_config.region() { + info!(region = region.as_ref()); + builder = builder.with_region(region.to_string()); + } + + let token = std::env::var_os("AWS_SESSION_TOKEN").and_then(|s| s.into_string().ok()); + builder = builder.with_credentials(Arc::new( + object_store::StaticCredentialProvider::new(object_store::aws::AwsCredential { + key_id, + secret_key, + token, + }), + )); + } + _ => { + let aws_config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await; + if let Some(region) = aws_config.region() { + info!(region = region.as_ref()); + builder = builder.with_region(region.to_string()); + } + if let Some(credentials_provider) = aws_config.credentials_provider() { + info!("using aws-config credentials provider"); + builder = builder.with_credentials(Arc::new(S3CredentialProvider { + aws: credentials_provider.clone(), + cache: tokio::sync::Mutex::new(None), + })); + } + } + } + builder +} + +// Keep credentials out of LiteArgs and its startup Debug log. Only supplied +// fields override the shared AWS configuration; credentials are replaced as a set. +struct WalS3Overrides { + endpoint: Option, + region: Option, + access_key_id: Option, + secret_access_key: Option, + session_token: Option, +} + +impl WalS3Overrides { + fn from_env() -> eyre::Result { + Self::from_getter(|name| match std::env::var(name) { + Ok(value) => Ok(Some(value)), + Err(std::env::VarError::NotPresent) => Ok(None), + Err(std::env::VarError::NotUnicode(_)) => { + Err(eyre::eyre!("{name} must contain valid UTF-8")) + } + }) + } + + fn from_getter(get: impl Fn(&str) -> eyre::Result>) -> eyre::Result { + let config = Self { + endpoint: get("S2LITE_WAL_AWS_ENDPOINT_URL_S3")?, + region: get("S2LITE_WAL_AWS_REGION")?, + access_key_id: get("S2LITE_WAL_AWS_ACCESS_KEY_ID")?, + secret_access_key: get("S2LITE_WAL_AWS_SECRET_ACCESS_KEY")?, + session_token: get("S2LITE_WAL_AWS_SESSION_TOKEN")?, + }; + eyre::ensure!( + config.access_key_id.is_some() == config.secret_access_key.is_some(), + "S2LITE_WAL_AWS_ACCESS_KEY_ID and S2LITE_WAL_AWS_SECRET_ACCESS_KEY must be set together" + ); + eyre::ensure!( + config.session_token.is_none() || config.access_key_id.is_some(), + "S2LITE_WAL_AWS_SESSION_TOKEN requires the WAL access key and secret key" + ); + Ok(config) + } + + fn apply( + self, + mut builder: object_store::aws::AmazonS3Builder, + ) -> object_store::aws::AmazonS3Builder { + if let Some(endpoint) = &self.endpoint { + // Override the S3-specific endpoint inherited from the main store. + builder = builder + .with_allow_http(endpoint.starts_with("http://")) + .with_config(object_store::aws::AmazonS3ConfigKey::S3Endpoint, endpoint); + } + if let Some(region) = self.region { + builder = builder.with_region(region); + } + if let (Some(key_id), Some(secret_key)) = (self.access_key_id, self.secret_access_key) { + builder = builder.with_credentials(Arc::new( + object_store::StaticCredentialProvider::new(object_store::aws::AwsCredential { + key_id, + secret_key, + token: self.session_token, + }), + )); + } + builder + } } async fn shutdown_signal(handle: axum_server::Handle) { @@ -459,7 +591,33 @@ impl object_store::CredentialProvider for S3CredentialProvider { #[cfg(test)] mod tests { - use super::{ServerProtocol, cli_endpoint, cli_env_hint}; + use super::{ServerProtocol, WalS3Overrides, cli_endpoint, cli_env_hint}; + + fn wal_config(values: &[(&str, &str)]) -> eyre::Result { + WalS3Overrides::from_getter(|name| { + Ok(values + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| (*value).to_owned())) + }) + } + + #[test] + fn wal_static_credentials_must_be_complete() { + for values in [ + vec![("S2LITE_WAL_AWS_ACCESS_KEY_ID", "test-key")], + vec![("S2LITE_WAL_AWS_SECRET_ACCESS_KEY", "do-not-log-this-secret")], + vec![("S2LITE_WAL_AWS_SESSION_TOKEN", "do-not-log-this-token")], + ] { + let error = wal_config(&values) + .err() + .expect("invalid credentials") + .to_string(); + assert!(error.contains("S2LITE_WAL_AWS_")); + assert!(!error.contains("do-not-log-this")); + } + assert!(wal_config(&[]).is_ok()); + } #[test] fn cli_endpoint_uses_localhost_with_explicit_port() { diff --git a/lite/tests/backend/common/mod.rs b/lite/tests/backend/common/mod.rs index 019728e9..a4bea287 100644 --- a/lite/tests/backend/common/mod.rs +++ b/lite/tests/backend/common/mod.rs @@ -3,6 +3,7 @@ use std::pin::Pin; use futures::Stream; use s2_common::{ basin::BasinName, + config::OptionalStreamConfig, encryption::EncryptionSpec, record::StreamPosition, stream::{AppendAck, AppendInput, StreamName}, @@ -30,6 +31,7 @@ pub async fn append( &basin, &stream, encryption.and_then(encryption_key_for_spec), + OptionalStreamConfig::default(), ) .await? .append(input) @@ -51,6 +53,7 @@ where &basin, &stream, encryption.and_then(encryption_key_for_spec), + OptionalStreamConfig::default(), ) .await? .append_session(inputs); diff --git a/lite/tests/backend/common/read.rs b/lite/tests/backend/common/read.rs index 28cb6b33..6e3405ba 100644 --- a/lite/tests/backend/common/read.rs +++ b/lite/tests/backend/common/read.rs @@ -3,6 +3,7 @@ use std::{pin::Pin, task::Poll, time::Duration}; use futures::StreamExt; use s2_common::{ basin::BasinName, + config::OptionalStreamConfig, encryption::EncryptionSpec, read_extent::{ReadLimit, ReadUntil}, record::{Record, SequencedRecord}, @@ -83,7 +84,12 @@ pub async fn try_open_read_session_with_encryption( ReadError, > { let read_session = backend - .open_for_read(basin, stream, encryption_key_for_spec(encryption)) + .open_for_read( + basin, + stream, + encryption_key_for_spec(encryption), + OptionalStreamConfig::default(), + ) .await? .read(start, end) .await?; diff --git a/lite/tests/backend/common/setup.rs b/lite/tests/backend/common/setup.rs index 324ea3b6..597f8023 100644 --- a/lite/tests/backend/common/setup.rs +++ b/lite/tests/backend/common/setup.rs @@ -1,4 +1,4 @@ -use std::{sync::Arc, time::Duration}; +use std::{fmt::Debug, future::Future, sync::Arc, time::Duration}; use bytes::Bytes; use bytesize::ByteSize; @@ -20,12 +20,16 @@ const TEST_AEGIS256_KEY: [u8; 32] = [0x42; 32]; const TEST_AES256_GCM_KEY: [u8; 32] = [0x24; 32]; pub async fn create_in_memory_db() -> Db { + create_in_memory_db_with_flush_interval(Some(Duration::from_millis(5))).await +} + +async fn create_in_memory_db_with_flush_interval(flush_interval: Option) -> Db { let object_store = Arc::new(InMemory::new()); let db_path = format!("/tmp/test_{}", Uuid::new_v4()); Db::builder(db_path, object_store) .with_settings(Settings { - flush_interval: Some(Duration::from_millis(5)), + flush_interval, ..Default::default() }) .build() @@ -38,6 +42,50 @@ pub async fn create_backend() -> Backend { Backend::new(db, ByteSize::mib(10)) } +pub async fn create_backend_without_auto_flush() -> (Backend, Db) { + let db = create_in_memory_db_with_flush_interval(None).await; + (Backend::new(db.clone(), ByteSize::mib(10)), db) +} + +/// Poll an operation until the database has a committed, unflushed write. +/// The database must have automatic flushing disabled and no unrelated writers. +pub async fn assert_pending_until_committed( + db: &Db, + operation: &mut (impl Future + Unpin), +) -> u64 { + tokio::time::timeout(Duration::from_secs(5), async { + tokio::select! { + biased; + result = operation => panic!("metadata acknowledged before flush: {result:?}"), + seq = async { + loop { + let seq = db.snapshot().await.unwrap().seq(); + if seq > db.status().durable_seq { + return seq; + } + tokio::task::yield_now().await; + } + } => seq, + } + }) + .await + .expect("metadata should be committed in memory before flushing") +} + +pub async fn assert_waits_for_flush(db: &Db, operation: impl Future) -> T { + tokio::pin!(operation); + tokio::time::timeout(Duration::from_secs(5), async { + let seq = assert_pending_until_committed(db, &mut operation).await; + assert!(futures::poll!(&mut operation).is_pending()); + db.flush().await.unwrap(); + let result = operation.await; + assert!(db.status().durable_seq >= seq); + result + }) + .await + .expect("metadata write should finish after an explicit flush") +} + pub fn test_basin_name(suffix: &str) -> BasinName { format!("test-basin-{}", suffix).parse().unwrap() } @@ -237,7 +285,12 @@ pub async fn append_payloads_with_encryption( fencing_token: None, }; backend - .open_for_append(basin, stream, encryption_key_for_spec(encryption)) + .open_for_append( + basin, + stream, + encryption_key_for_spec(encryption), + OptionalStreamConfig::default(), + ) .await .expect("Failed to open append handle") .append(input) @@ -257,7 +310,7 @@ pub async fn append_timestamped_payloads( fencing_token: None, }; backend - .open_for_append(basin, stream, None) + .open_for_append(basin, stream, None, OptionalStreamConfig::default()) .await .expect("Failed to open append handle") .append(input) diff --git a/lite/tests/backend/control_plane/basin.rs b/lite/tests/backend/control_plane/basin.rs index f5d09f22..67e1037d 100644 --- a/lite/tests/backend/control_plane/basin.rs +++ b/lite/tests/backend/control_plane/basin.rs @@ -4,8 +4,7 @@ use s2_common::{ basin::{BasinNamePrefix, BasinNameStartAfter, ListBasinsRequest}, config::{ BasinConfig, BasinReconfiguration, OptionalDeleteOnEmptyConfig, OptionalStreamConfig, - RetentionPolicy, StorageClass, StreamReconfiguration, TimestampingMode, - TimestampingReconfiguration, + RetentionPolicy, StreamReconfiguration, TimestampingMode, TimestampingReconfiguration, }, maybe::Maybe, resources::{ProvisionMode, ProvisionResult, RequestToken}, @@ -16,6 +15,80 @@ use s2_lite::backend::error::{ use super::common::*; +#[tokio::test] +async fn test_provision_basin_acknowledges_only_durable_metadata() { + let (backend, db) = create_backend_without_auto_flush().await; + let basin = test_basin_name("durable"); + let result = assert_waits_for_flush( + &db, + backend.provision_basin(basin.clone(), BasinConfig::default(), ProvisionMode::Ensure), + ) + .await + .unwrap(); + assert!(matches!(result, ProvisionResult::Created(_))); + assert_eq!( + backend.get_basin_config(basin).await.unwrap(), + BasinConfig::default() + ); + backend.close().await.unwrap(); +} + +#[rstest::rstest] +#[case::ensure(ProvisionMode::Ensure, false)] +#[case::idempotent_create(ProvisionMode::CreateOnly { + request_token: Some("original".parse().unwrap()), +}, false)] +#[case::create_without_token(ProvisionMode::CreateOnly { + request_token: None, +}, true)] +#[case::create_with_different_token(ProvisionMode::CreateOnly { + request_token: Some("different".parse().unwrap()), +}, true)] +#[tokio::test] +async fn test_basin_retries_acknowledge_only_durable_metadata( + #[case] retry_mode: ProvisionMode, + #[case] expect_already_exists: bool, +) { + let (backend, db) = create_backend_without_auto_flush().await; + let basin = test_basin_name("durable-retry"); + let creation = backend.provision_basin( + basin.clone(), + BasinConfig::default(), + ProvisionMode::CreateOnly { + request_token: Some("original".parse().unwrap()), + }, + ); + tokio::pin!(creation); + assert_pending_until_committed(&db, &mut creation).await; + assert!(matches!( + backend.get_basin_config(basin.clone()).await, + Err(GetBasinConfigError::BasinNotFound(_)) + )); + + let retry = assert_waits_for_flush( + &db, + backend.provision_basin(basin.clone(), BasinConfig::default(), retry_mode), + ) + .await; + if expect_already_exists { + assert!(matches!( + retry, + Err(ProvisionBasinError::BasinAlreadyExists(_)) + )); + } else { + assert!(matches!(retry, Ok(ProvisionResult::Noop(_)))); + } + assert!(matches!( + creation.await.unwrap(), + ProvisionResult::Created(_) + )); + assert_eq!( + backend.get_basin_config(basin).await.unwrap(), + BasinConfig::default() + ); + backend.close().await.unwrap(); +} + #[tokio::test] async fn test_create_basin_idempotency_respects_request_token() { let backend = create_backend().await; @@ -167,7 +240,7 @@ async fn test_provision_basin_ensure_updates_config() { let mut updated_config = initial_config.clone(); updated_config.create_stream_on_append = true; updated_config.create_stream_on_read = true; - updated_config.default_stream_config.storage_class = Some(StorageClass::Standard); + updated_config.default_stream_config.storage_class = Some("standard".into()); backend .provision_basin( @@ -186,7 +259,7 @@ async fn test_provision_basin_ensure_updates_config() { assert!(stored_config.create_stream_on_read); assert_eq!( stored_config.default_stream_config.storage_class, - Some(StorageClass::Standard) + Some("standard".into()) ); backend @@ -209,7 +282,7 @@ async fn test_provision_basin_ensure_resets_unspecified_config() { create_stream_on_append: true, create_stream_on_read: false, default_stream_config: OptionalStreamConfig { - storage_class: Some(StorageClass::Standard), + storage_class: Some("standard".into()), retention_policy: Some(RetentionPolicy::Infinite()), ..Default::default() }, @@ -296,7 +369,7 @@ async fn test_reconfigure_basin_updates_nested_defaults() { let backend = create_backend().await; let basin_name = test_basin_name("basin-reconfigure"); let mut initial_config = BasinConfig::default(); - initial_config.default_stream_config.storage_class = Some(StorageClass::Standard); + initial_config.default_stream_config.storage_class = Some("standard".into()); backend .provision_basin( @@ -314,7 +387,7 @@ async fn test_reconfigure_basin_updates_nested_defaults() { ..Default::default() }; let mut stream_reconfig = StreamReconfiguration { - storage_class: Maybe::from(Some(StorageClass::Express)), + storage_class: Maybe::from(Some("express".into())), retention_policy: Maybe::from(Some(RetentionPolicy::Infinite())), ..Default::default() }; @@ -336,7 +409,7 @@ async fn test_reconfigure_basin_updates_nested_defaults() { assert!(updated.create_stream_on_read); assert_eq!( updated.default_stream_config.storage_class, - Some(StorageClass::Express) + Some("express".into()) ); assert_eq!( updated.default_stream_config.retention_policy, @@ -353,7 +426,7 @@ async fn test_reconfigure_basin_updates_nested_defaults() { .expect("Failed to fetch basin config after reconfigure"); assert_eq!( fetched.default_stream_config.storage_class, - Some(StorageClass::Express) + Some("express".into()) ); assert_eq!( fetched.default_stream_config.retention_policy, @@ -367,6 +440,30 @@ async fn test_reconfigure_basin_updates_nested_defaults() { assert!(fetched.create_stream_on_read); } +#[tokio::test(start_paused = true)] +async fn test_delete_basin_retry_waits_for_durable_metadata() { + let (backend, db) = create_backend_without_auto_flush().await; + let basin = assert_waits_for_flush( + &db, + create_test_basin(&backend, "durable-delete", BasinConfig::default()), + ) + .await; + + let mut first = Box::pin(backend.delete_basin(basin.clone())); + assert_pending_until_committed(&db, &mut first).await; + drop(first); + + let retry = backend.delete_basin(basin); + tokio::pin!(retry); + assert!( + tokio::time::timeout(Duration::from_secs(1), &mut retry) + .await + .is_err() + ); + assert_waits_for_flush(&db, retry).await.unwrap(); + backend.close().await.unwrap(); +} + #[tokio::test] async fn test_delete_basin_marks_deleting_and_blocks_create() { let backend = create_backend().await; diff --git a/lite/tests/backend/control_plane/stream.rs b/lite/tests/backend/control_plane/stream.rs index a3ca62c5..6ded3547 100644 --- a/lite/tests/backend/control_plane/stream.rs +++ b/lite/tests/backend/control_plane/stream.rs @@ -5,8 +5,7 @@ use s2_common::{ config::{ BasinConfig, BasinReconfiguration, DeleteOnEmptyReconfiguration, OptionalDeleteOnEmptyConfig, OptionalStreamConfig, OptionalTimestampingConfig, - RetentionPolicy, StorageClass, StreamReconfiguration, TimestampingMode, - TimestampingReconfiguration, + RetentionPolicy, StreamReconfiguration, TimestampingMode, TimestampingReconfiguration, }, encryption::EncryptionAlgorithm, maybe::Maybe, @@ -23,6 +22,34 @@ use s2_lite::backend::error::{ use super::common::*; +#[tokio::test] +async fn test_provision_stream_acknowledges_only_durable_metadata() { + let (backend, db) = create_backend_without_auto_flush().await; + let basin = test_basin_name("durable-stream"); + let stream = test_stream_name("durable"); + assert_waits_for_flush( + &db, + backend.provision_basin(basin.clone(), BasinConfig::default(), ProvisionMode::Ensure), + ) + .await + .unwrap(); + + let result = assert_waits_for_flush( + &db, + backend.provision_stream( + basin.clone(), + stream.clone(), + OptionalStreamConfig::default(), + ProvisionMode::Ensure, + ), + ) + .await + .unwrap(); + assert!(matches!(result, ProvisionResult::Created(_))); + backend.get_stream_config(basin, stream).await.unwrap(); + backend.close().await.unwrap(); +} + #[tokio::test] async fn test_create_stream_honors_basin_defaults() { let backend = create_backend().await; @@ -30,7 +57,7 @@ async fn test_create_stream_honors_basin_defaults() { let basin_config = BasinConfig { default_stream_config: OptionalStreamConfig { - storage_class: Some(StorageClass::Standard), + storage_class: Some("standard".into()), retention_policy: Some(RetentionPolicy::Infinite()), timestamping: OptionalTimestampingConfig { mode: Some(TimestampingMode::ClientRequire), @@ -70,7 +97,7 @@ async fn test_create_stream_honors_basin_defaults() { .get_stream_config(basin_name, stream_name) .await .expect("Failed to fetch stream config"); - assert_eq!(config.storage_class, StorageClass::Standard); + assert_eq!(config.storage_class.as_deref(), Some("standard")); assert_eq!(config.retention_policy, RetentionPolicy::Infinite()); assert_eq!(config.timestamping.mode, TimestampingMode::ClientRequire); } @@ -212,7 +239,7 @@ async fn test_create_stream_idempotency_and_request_token() { let stream_name = test_stream_name("stream-idempotency"); let config = OptionalStreamConfig { - storage_class: Some(StorageClass::Express), + storage_class: Some("express".into()), ..Default::default() }; @@ -234,7 +261,7 @@ async fn test_create_stream_idempotency_and_request_token() { .get_stream_config(basin_name.clone(), stream_name.clone()) .await .expect("Failed to fetch stored stream config"); - assert_eq!(stored_config.storage_class, StorageClass::Express); + assert_eq!(stored_config.storage_class.as_deref(), Some("express")); let idempotent = backend .provision_stream( @@ -298,7 +325,7 @@ async fn test_provision_stream_ensure_preserves_idempotency_key() { let stream_name = test_stream_name("stream-idempotency-key-preserve"); let config = OptionalStreamConfig { - storage_class: Some(StorageClass::Standard), + storage_class: Some("standard".into()), ..Default::default() }; let token: RequestToken = "stream-token-preserve".parse().unwrap(); @@ -347,7 +374,7 @@ async fn test_provision_stream_ensure_preserves_idempotency_key() { .get_stream_config(basin_name.clone(), stream_name.clone()) .await .expect("Failed to fetch stream config"); - assert_eq!(stored_config.storage_class, StorageClass::Express); + assert_eq!(stored_config.storage_class.as_deref(), Some("express")); assert_eq!(stored_config.timestamping.mode, TimestampingMode::Arrival); backend @@ -371,7 +398,7 @@ async fn test_provision_stream_ensure_noops_when_effective_config_matches() { "stream-ensure-effective-noop", BasinConfig { default_stream_config: OptionalStreamConfig { - storage_class: Some(StorageClass::Express), + storage_class: Some("express".into()), retention_policy: Some(RetentionPolicy::Age(Duration::from_secs( 10 * 24 * 60 * 60, ))), @@ -383,7 +410,7 @@ async fn test_provision_stream_ensure_noops_when_effective_config_matches() { .await; let stream_name = test_stream_name("stream-ensure-effective-noop"); let config = OptionalStreamConfig { - storage_class: Some(StorageClass::Standard), + storage_class: Some("standard".into()), retention_policy: Some(RetentionPolicy::Infinite()), ..Default::default() }; @@ -489,7 +516,7 @@ async fn test_provision_stream_idempotency_ignores_changed_basin_defaults() { basin_name.clone(), BasinReconfiguration { default_stream_config: Maybe::from(Some(StreamReconfiguration { - storage_class: Maybe::from(Some(StorageClass::Standard)), + storage_class: Maybe::from(Some("standard".into())), ..Default::default() })), ..Default::default() @@ -517,7 +544,7 @@ async fn test_reconfigure_stream_updates_selected_fields() { let basin_name = test_basin_name("stream-reconfigure"); let mut basin_config = BasinConfig::default(); - basin_config.default_stream_config.storage_class = Some(StorageClass::Standard); + basin_config.default_stream_config.storage_class = Some("standard".into()); backend .provision_basin( @@ -557,7 +584,7 @@ async fn test_reconfigure_stream_updates_selected_fields() { uncapped: Maybe::from(Some(true)), }; let mut stream_reconfig = StreamReconfiguration { - storage_class: Maybe::from(Some(StorageClass::Express)), + storage_class: Maybe::from(Some("express".into())), retention_policy: Maybe::from(Some(RetentionPolicy::Infinite())), ..Default::default() }; @@ -568,7 +595,7 @@ async fn test_reconfigure_stream_updates_selected_fields() { .await .expect("Failed to reconfigure stream"); - assert_eq!(updated.storage_class, StorageClass::Express); + assert_eq!(updated.storage_class.as_deref(), Some("express")); assert_eq!(updated.retention_policy, RetentionPolicy::Infinite()); assert_eq!(updated.timestamping.mode, TimestampingMode::Arrival); assert!(updated.timestamping.uncapped); @@ -577,7 +604,7 @@ async fn test_reconfigure_stream_updates_selected_fields() { .get_stream_config(basin_name, stream_name) .await .expect("Failed to fetch stream config after reconfigure"); - assert_eq!(fetched.storage_class, StorageClass::Express); + assert_eq!(fetched.storage_class.as_deref(), Some("express")); assert_eq!(fetched.retention_policy, RetentionPolicy::Infinite()); assert_eq!(fetched.timestamping.mode, TimestampingMode::Arrival); assert!(fetched.timestamping.uncapped); @@ -590,7 +617,7 @@ async fn test_reconfigure_stream_clears_fields_to_basin_defaults() { let basin_config = BasinConfig { default_stream_config: OptionalStreamConfig { - storage_class: Some(StorageClass::Standard), + storage_class: Some("standard".into()), retention_policy: Some(RetentionPolicy::Infinite()), timestamping: OptionalTimestampingConfig { mode: Some(TimestampingMode::Arrival), @@ -616,7 +643,7 @@ async fn test_reconfigure_stream_clears_fields_to_basin_defaults() { let stream_name = test_stream_name("stream-reconfigure-clear-defaults"); let stream_config = OptionalStreamConfig { - storage_class: Some(StorageClass::Express), + storage_class: Some("express".into()), retention_policy: Some(RetentionPolicy::Age(Duration::from_secs(60))), timestamping: OptionalTimestampingConfig { mode: Some(TimestampingMode::ClientRequire), @@ -656,7 +683,7 @@ async fn test_reconfigure_stream_clears_fields_to_basin_defaults() { .await .expect("Failed to reconfigure stream"); - assert_eq!(updated.storage_class, StorageClass::Standard); + assert_eq!(updated.storage_class.as_deref(), Some("standard")); assert_eq!(updated.retention_policy, RetentionPolicy::Infinite()); assert_eq!(updated.timestamping.mode, TimestampingMode::Arrival); assert!(updated.timestamping.uncapped); @@ -670,84 +697,101 @@ async fn test_reconfigure_stream_clears_fields_to_basin_defaults() { assert_eq!(fetched, updated); } +#[rstest::rstest] +#[case::patch(false, false)] +#[case::ensure(true, false)] +#[case::cancelled_patch(false, true)] +#[case::cancelled_ensure(true, true)] #[tokio::test] -async fn test_reconfigure_stream_updates_active_streamer() { - let (backend, basin_name, stream_name) = setup_backend_with_stream( - "stream-reconfigure-active", - "stream", - OptionalStreamConfig::default(), +async fn test_stream_config_updates_active_streamer(#[case] ensure: bool, #[case] cancel: bool) { + let (backend, db) = create_backend_without_auto_flush().await; + let basin = assert_waits_for_flush( + &db, + create_test_basin(&backend, "config-delivery", BasinConfig::default()), ) .await; - - append_payloads(&backend, &basin_name, &stream_name, &[b"seed"]).await; - - let ts_reconfig = TimestampingReconfiguration { - mode: Maybe::from(Some(TimestampingMode::ClientRequire)), - uncapped: Maybe::default(), - }; - let reconfig = StreamReconfiguration { - timestamping: Maybe::from(Some(ts_reconfig)), - ..Default::default() - }; - - backend - .reconfigure_stream(basin_name.clone(), stream_name.clone(), reconfig) - .await - .expect("Failed to reconfigure stream"); - - check_tail(&backend, basin_name.clone(), stream_name.clone()) - .await - .expect("Failed to check tail"); - - let input = AppendInput { - records: create_test_record_batch(vec![Bytes::from_static(b"missing timestamp")]), - match_seq_num: None, - fencing_token: None, - }; - let result = append(&backend, basin_name, stream_name, input, None).await; - assert!(matches!(result, Err(AppendError::TimestampMissing(_)))); -} - -#[tokio::test] -async fn test_provision_stream_ensure_updates_active_streamer() { - let (backend, basin_name, stream_name) = setup_backend_with_stream( - "stream-ensure-active", - "stream", - OptionalStreamConfig::default(), + let stream = assert_waits_for_flush( + &db, + create_test_stream( + &backend, + &basin, + "stream", + OptionalStreamConfig { + timestamping: OptionalTimestampingConfig { + mode: Some(TimestampingMode::ClientRequire), + ..Default::default() + }, + ..Default::default() + }, + ), ) .await; - - append_payloads(&backend, &basin_name, &stream_name, &[b"seed"]).await; - - let config = OptionalStreamConfig { - timestamping: OptionalTimestampingConfig { - mode: Some(TimestampingMode::ClientRequire), - ..Default::default() - }, - ..Default::default() - }; - - backend - .provision_stream( - basin_name.clone(), - stream_name.clone(), - config, - ProvisionMode::Ensure, - ) + let _active_streamer = backend + .open_for_append(&basin, &stream, None, OptionalStreamConfig::default()) .await - .expect("Ensure should succeed for an existing stream"); + .unwrap(); - check_tail(&backend, basin_name.clone(), stream_name.clone()) - .await - .expect("Failed to check tail"); + let mut update = Box::pin(async { + if ensure { + backend + .provision_stream( + basin.clone(), + stream.clone(), + OptionalStreamConfig { + timestamping: OptionalTimestampingConfig { + mode: Some(TimestampingMode::Arrival), + ..Default::default() + }, + ..Default::default() + }, + ProvisionMode::Ensure, + ) + .await + .unwrap(); + } else { + backend + .reconfigure_stream( + basin.clone(), + stream.clone(), + StreamReconfiguration { + timestamping: Maybe::from(Some(TimestampingReconfiguration { + mode: Maybe::from(Some(TimestampingMode::Arrival)), + ..Default::default() + })), + ..Default::default() + }, + ) + .await + .unwrap(); + } + }); + assert_pending_until_committed(&db, &mut update).await; + if cancel { + drop(update); + db.flush().await.unwrap(); + } else { + db.flush().await.unwrap(); + update.await; + } - let input = AppendInput { - records: create_test_record_batch(vec![Bytes::from_static(b"missing timestamp")]), - match_seq_num: None, - fencing_token: None, + // Application is asynchronous; retry until the active streamer accepts arrival timestamps. + let append_after_update = async { + loop { + let input = AppendInput { + records: create_test_record_batch(vec![Bytes::from_static(b"no timestamp")]), + match_seq_num: None, + fencing_token: None, + }; + match append(&backend, basin.clone(), stream.clone(), input, None).await { + Err(AppendError::TimestampMissing(_)) => tokio::task::yield_now().await, + result => break result, + } + } }; - let result = append(&backend, basin_name, stream_name, input, None).await; - assert!(matches!(result, Err(AppendError::TimestampMissing(_)))); + assert_waits_for_flush(&db, append_after_update) + .await + .unwrap(); + backend.close().await.unwrap(); } #[tokio::test] @@ -779,6 +823,38 @@ async fn test_create_stream_fails_when_basin_deleting() { )); } +#[tokio::test(start_paused = true)] +async fn test_delete_stream_retry_waits_for_durable_metadata() { + let (backend, db) = create_backend_without_auto_flush().await; + let basin = assert_waits_for_flush( + &db, + create_test_basin(&backend, "durable-delete", BasinConfig::default()), + ) + .await; + let stream = assert_waits_for_flush( + &db, + create_test_stream(&backend, &basin, "durable-delete", Default::default()), + ) + .await; + + let mut first = Box::pin(backend.delete_stream(basin.clone(), stream.clone())); + // Stream deletion first persists the terminal trim, then the metadata marker. + assert_pending_until_committed(&db, &mut first).await; + db.flush().await.unwrap(); + assert_pending_until_committed(&db, &mut first).await; + drop(first); + + let retry = backend.delete_stream(basin, stream); + tokio::pin!(retry); + assert!( + tokio::time::timeout(Duration::from_secs(1), &mut retry) + .await + .is_err() + ); + assert_waits_for_flush(&db, retry).await.unwrap(); + backend.close().await.unwrap(); +} + #[tokio::test] async fn test_delete_stream_marks_deleted_and_blocks_recreation() { let backend = create_backend().await; diff --git a/lite/tests/backend/data_plane/auto_create.rs b/lite/tests/backend/data_plane/auto_create.rs index 24bdfc00..491a3eb7 100644 --- a/lite/tests/backend/data_plane/auto_create.rs +++ b/lite/tests/backend/data_plane/auto_create.rs @@ -3,7 +3,10 @@ use std::time::Duration; use bytes::Bytes; use s2_common::{ basin::BasinName, - config::BasinConfig, + config::{ + BasinConfig, DeleteOnEmptyConfig, OptionalDeleteOnEmptyConfig, OptionalStreamConfig, + RetentionPolicy, StreamConfig, + }, encryption::EncryptionAlgorithm, read_extent::{ReadLimit, ReadUntil}, record::StreamPosition, @@ -123,6 +126,140 @@ async fn test_backend_append_auto_creates_stream_with_basin_cipher() { assert_eq!(envelope_bodies(&records), vec![b"secret".to_vec()]); } +fn basin_config_with_defaults() -> BasinConfig { + BasinConfig { + create_stream_on_append: true, + default_stream_config: OptionalStreamConfig { + storage_class: Some("standard".into()), + retention_policy: Some(RetentionPolicy::Age(Duration::from_secs(7 * 24 * 60 * 60))), + ..Default::default() + }, + ..Default::default() + } +} + +fn requested_stream_config() -> OptionalStreamConfig { + OptionalStreamConfig { + retention_policy: Some(RetentionPolicy::Age(Duration::from_secs(3600))), + delete_on_empty: OptionalDeleteOnEmptyConfig { + min_age: Some(Duration::from_secs(300)), + }, + ..Default::default() + } +} + +fn expected_merged_stream_config() -> StreamConfig { + StreamConfig { + storage_class: Some("standard".into()), + retention_policy: RetentionPolicy::Age(Duration::from_secs(3600)), + timestamping: Default::default(), + delete_on_empty: DeleteOnEmptyConfig { + min_age: Duration::from_secs(300), + }, + } +} + +#[tokio::test] +async fn test_backend_append_auto_create_applies_stream_config() { + let backend = create_backend().await; + let basin_name = create_test_basin( + &backend, + "backend-auto-create-config", + basin_config_with_defaults(), + ) + .await; + let stream_name = test_stream_name("missing"); + + let input = AppendInput { + records: create_test_record_batch(vec![Bytes::from_static(b"hello")]), + match_seq_num: None, + fencing_token: None, + }; + let ack = backend + .open_for_append(&basin_name, &stream_name, None, requested_stream_config()) + .await + .expect("Failed to open append handle") + .append(input) + .await + .expect("Failed to append to auto-created stream"); + assert_eq!(ack.end.seq_num, 1); + + let config = backend + .get_stream_config(basin_name.clone(), stream_name.clone()) + .await + .expect("Failed to get stream config"); + assert_eq!(config, expected_merged_stream_config()); +} + +#[tokio::test] +async fn test_backend_append_ignores_stream_config_for_existing_stream() { + let backend = create_backend().await; + let basin_name = create_test_basin( + &backend, + "backend-auto-create-config-existing", + basin_config_with_defaults(), + ) + .await; + let stream_name = create_test_stream( + &backend, + &basin_name, + "existing", + OptionalStreamConfig::default(), + ) + .await; + let before = backend + .get_stream_config(basin_name.clone(), stream_name.clone()) + .await + .expect("Failed to get stream config"); + + let input = AppendInput { + records: create_test_record_batch(vec![Bytes::from_static(b"hello")]), + match_seq_num: None, + fencing_token: None, + }; + backend + .open_for_append(&basin_name, &stream_name, None, requested_stream_config()) + .await + .expect("Failed to open append handle") + .append(input) + .await + .expect("Failed to append to existing stream"); + + let after = backend + .get_stream_config(basin_name.clone(), stream_name.clone()) + .await + .expect("Failed to get stream config"); + assert_eq!(after, before); + assert_ne!(after, expected_merged_stream_config()); +} + +#[tokio::test] +async fn test_backend_read_auto_create_applies_stream_config() { + let backend = create_backend().await; + let basin_name = create_test_basin( + &backend, + "backend-auto-create-read-config", + BasinConfig { + create_stream_on_append: false, + create_stream_on_read: true, + ..basin_config_with_defaults() + }, + ) + .await; + let stream_name = test_stream_name("missing"); + + backend + .open_for_read(&basin_name, &stream_name, None, requested_stream_config()) + .await + .expect("Failed to open read handle on auto-created stream"); + + let config = backend + .get_stream_config(basin_name.clone(), stream_name.clone()) + .await + .expect("Failed to get stream config"); + assert_eq!(config, expected_merged_stream_config()); +} + #[tokio::test] async fn test_backend_append_without_auto_create_returns_not_found() { let backend = create_backend().await; diff --git a/lite/tests/backend/data_plane/mixed.rs b/lite/tests/backend/data_plane/mixed.rs index 63b9a372..7bc8c40c 100644 --- a/lite/tests/backend/data_plane/mixed.rs +++ b/lite/tests/backend/data_plane/mixed.rs @@ -2,7 +2,7 @@ use std::{sync::Arc, time::Duration}; use bytes::Bytes; use s2_common::{ - config::{OptionalStreamConfig, RetentionPolicy, StorageClass, StreamReconfiguration}, + config::{OptionalStreamConfig, RetentionPolicy, StreamReconfiguration}, read_extent::{ReadLimit, ReadUntil}, stream::{AppendInput, ReadEnd, ReadFrom, ReadStart}, }; @@ -147,7 +147,7 @@ async fn test_concurrent_reconfigure_during_append() { ready.notified().await; let reconfig = StreamReconfiguration { - storage_class: s2_common::maybe::Maybe::from(Some(StorageClass::Express)), + storage_class: s2_common::maybe::Maybe::from(Some("express".into())), retention_policy: s2_common::maybe::Maybe::from(Some(RetentionPolicy::Infinite())), timestamping: s2_common::maybe::Maybe::default(), delete_on_empty: s2_common::maybe::Maybe::default(), @@ -157,7 +157,7 @@ async fn test_concurrent_reconfigure_during_append() { .reconfigure_stream(basin_name.clone(), stream_name.clone(), reconfig) .await .expect("Failed to reconfigure stream during appends"); - assert_eq!(updated_config.storage_class, StorageClass::Express); + assert_eq!(updated_config.storage_class.as_deref(), Some("express")); append_handle.await.unwrap(); diff --git a/lite/tests/backend/data_plane/read.rs b/lite/tests/backend/data_plane/read.rs index 900d89ee..511ae50c 100644 --- a/lite/tests/backend/data_plane/read.rs +++ b/lite/tests/backend/data_plane/read.rs @@ -326,6 +326,31 @@ async fn test_read_at_tail_without_follow_returns_unwritten( } } +#[rstest] +#[case::seq_num(ReadFrom::SeqNum(0))] +#[case::timestamp(ReadFrom::Timestamp(0))] +#[tokio::test] +async fn test_read_at_tail_of_empty_stream_returns_unwritten(#[case] from: ReadFrom) { + let (backend, basin_name, stream_name) = setup_backend_with_stream( + "read-empty-at-tail", + "stream", + OptionalStreamConfig::default(), + ) + .await; + + let start = ReadStart { from, clamp: false }; + let end = tail_read_end(TailEndCase::CountNoWait); + let result = try_open_read_session(&backend, &basin_name, &stream_name, start, end).await; + + match result { + Err(ReadError::Unwritten(UnwrittenError(tail))) => { + assert_eq!(tail, StreamPosition::MIN); + } + Ok(_) => panic!("Expected Unwritten error for {from:?} on an empty stream, got Ok"), + Err(e) => panic!("Expected Unwritten error for {from:?} on an empty stream, got: {e:?}"), + } +} + #[tokio::test] async fn test_read_from_tail_offset() { let (backend, basin_name, stream_name) = setup_backend_with_stream( diff --git a/resource-spec/CHANGELOG.md b/resource-spec/CHANGELOG.md index 745d023e..d5368b6d 100644 --- a/resource-spec/CHANGELOG.md +++ b/resource-spec/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. +## [0.3.0] - 2026-09-25 + +### Features + +- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) + + + ## [0.2.2] - 2026-07-07 ### Miscellaneous Tasks diff --git a/resource-spec/Cargo.toml b/resource-spec/Cargo.toml index 1c0ca47d..4c5816c3 100644 --- a/resource-spec/Cargo.toml +++ b/resource-spec/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-resource-spec" -version = "0.2.2" +version = "0.3.0" description = "Declarative resource specifications for S2" edition.workspace = true license.workspace = true @@ -9,6 +9,7 @@ homepage.workspace = true keywords = ["s2", "durable", "streams", "config", "schema"] [dependencies] +compact_str = { workspace = true, features = ["serde"] } humantime = { workspace = true } s2-common = { workspace = true } schemars = { workspace = true } diff --git a/resource-spec/src/lib.rs b/resource-spec/src/lib.rs index 7a0ea7fa..042edd6c 100644 --- a/resource-spec/src/lib.rs +++ b/resource-spec/src/lib.rs @@ -2,6 +2,7 @@ use std::{borrow::Cow, time::Duration}; +use compact_str::CompactString; use s2_common::{basin::BasinName, stream::StreamName}; use serde::{Deserialize, Serialize}; @@ -50,9 +51,10 @@ pub struct BasinConfig { #[derive(Debug, Clone, Deserialize, Default, schemars::JsonSchema)] #[serde(deny_unknown_fields)] pub struct StreamConfig { - /// Storage class for recent writes. + /// [Storage class](https://s2.dev/docs/storage-classes) for recent writes. #[serde(default)] - pub storage_class: Option, + #[schemars(with = "Option")] + pub storage_class: Option, /// Retention policy for the stream. If unspecified, the default is to retain records for 7 /// days. #[serde(default)] @@ -65,36 +67,6 @@ pub struct StreamConfig { pub delete_on_empty: Option, } -#[derive(Debug, Clone, Deserialize, Serialize)] -#[serde(rename_all = "kebab-case")] -pub enum StorageClass { - Standard, - Express, -} - -impl schemars::JsonSchema for StorageClass { - fn schema_name() -> Cow<'static, str> { - "StorageClass".into() - } - - fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema { - schemars::json_schema!({ - "type": "string", - "description": "Storage class for recent writes.", - "enum": ["standard", "express"] - }) - } -} - -impl From for s2_common::config::StorageClass { - fn from(s: StorageClass) -> Self { - match s { - StorageClass::Standard => Self::Standard, - StorageClass::Express => Self::Express, - } - } -} - #[derive(Debug, Clone, Deserialize, Serialize)] pub enum EncryptionAlgorithm { #[serde(rename = "aegis-256")] @@ -287,7 +259,7 @@ impl From for s2_common::config::OptionalDeleteOnEmptyConfig { impl From for s2_common::config::OptionalStreamConfig { fn from(s: StreamConfig) -> Self { Self { - storage_class: s.storage_class.map(Into::into), + storage_class: s.storage_class, retention_policy: s.retention_policy.map(|rp| rp.0), timestamping: s.timestamping.map(Into::into).unwrap_or_default(), delete_on_empty: s.delete_on_empty.map(Into::into).unwrap_or_default(), @@ -351,8 +323,9 @@ pub fn validate(spec: &Resources) -> Result<(), String> { } fn validate_stream_config(config: &StreamConfig, context: &str, errors: &mut Vec) { - let config = s2_common::config::OptionalStreamConfig::from(config.clone()); - if let Err(err) = config.validate() { + if let Some(retention_policy) = config.retention_policy + && let Err(err) = retention_policy.0.validate() + { errors.push(format!("{context}: {err}")); } } @@ -459,7 +432,7 @@ mod tests { assert_eq!(config.create_stream_on_read, Some(false)); let dsc = config.default_stream_config.as_ref().unwrap(); - assert!(matches!(dsc.storage_class, Some(StorageClass::Express))); + assert_eq!(dsc.storage_class.as_deref(), Some("express")); assert!(matches!( dsc.retention_policy.as_ref().map(|r| &r.0), Some(s2_common::config::RetentionPolicy::Age(_)) @@ -479,7 +452,7 @@ mod tests { let stream = &basin.streams[0]; assert_eq!(stream.name.as_ref(), "events"); let sc = stream.config.as_ref().unwrap(); - assert!(matches!(sc.storage_class, Some(StorageClass::Standard))); + assert_eq!(sc.storage_class.as_deref(), Some("standard")); assert!(matches!( sc.retention_policy.as_ref().map(|r| &r.0), Some(s2_common::config::RetentionPolicy::Infinite()) @@ -604,7 +577,7 @@ mod tests { #[test] fn stream_config_conversion() { let spec = StreamConfig { - storage_class: Some(StorageClass::Standard), + storage_class: Some("standard".into()), retention_policy: Some(RetentionPolicy( s2_common::config::RetentionPolicy::Infinite(), )), @@ -612,10 +585,7 @@ mod tests { delete_on_empty: None, }; let config = s2_common::config::OptionalStreamConfig::from(spec); - assert_eq!( - config.storage_class, - Some(s2_common::config::StorageClass::Standard) - ); + assert_eq!(config.storage_class, Some("standard".into())); assert_eq!( config.retention_policy, Some(s2_common::config::RetentionPolicy::Infinite()) diff --git a/sdk/CHANGELOG.md b/sdk/CHANGELOG.md index 8d07c476..ed7c6a3b 100644 --- a/sdk/CHANGELOG.md +++ b/sdk/CHANGELOG.md @@ -2,6 +2,58 @@ All notable changes to this project will be documented in this file. +## [0.35.0] - 2026-09-25 + +### Features + +- [**breaking**] Expose storage classes as strings and in location responses ([#775](https://github.com/s2-streamstore/s2/issues/775)) + + + +## [0.34.10] - 2026-09-24 + +### Bug Fixes + +- Preserve uncertainty across append retries ([#767](https://github.com/s2-streamstore/s2/issues/767)) + + + +## [0.34.9] - 2026-09-22 + + + +## [0.34.8] - 2026-09-16 + +### Bug Fixes + +- Reject Content-Type in default headers ([#740](https://github.com/s2-streamstore/s2/issues/740)) + + + +## [0.34.7] - 2026-09-11 + +### Features + +- `s2-stream-config` header for auto-created streams ([#718](https://github.com/s2-streamstore/s2/issues/718)) + + + +## [0.34.6] - 2026-09-11 + +### Features + +- Set default request headers (`_hidden` only) ([#731](https://github.com/s2-streamstore/s2/issues/731)) + + + +## [0.34.5] - 2026-09-10 + +### Bug Fixes + +- Reject NUL bytes in stream names and access token IDs ([#728](https://github.com/s2-streamstore/s2/issues/728)) + + + ## [0.34.4] - 2026-09-01 ### Features diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml index ecdd764f..db739f37 100644 --- a/sdk/Cargo.toml +++ b/sdk/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "s2-sdk" description = "Rust SDK for S2" -version = "0.34.4" +version = "0.35.0" edition.workspace = true license.workspace = true repository = "https://github.com/s2-streamstore/s2/tree/main/sdk" @@ -25,6 +25,7 @@ async-compression = { version = "0.4", features = ["tokio", "gzip", "zstd"] } async-stream = { workspace = true } async-trait = { workspace = true } bytes = { workspace = true } +compact_str = { workspace = true, features = ["serde"] } futures-core = { workspace = true } futures-util = { workspace = true } h2 = "0.4" @@ -46,7 +47,7 @@ serde_json = { workspace = true } serde_urlencoded = "0.7" thiserror = { workspace = true } time = { workspace = true } -tokio = { version = "1.6", features = ["time", "macros", "io-util", "sync"] } +tokio = { version = "1.53", features = ["time", "macros", "io-util", "sync"] } tokio-muxt = "0.7" tokio-stream = { workspace = true } tokio-util = { workspace = true, features = ["rt", "codec"] } diff --git a/sdk/examples/reconfigure_basin.rs b/sdk/examples/reconfigure_basin.rs index 45a1488d..a1946d28 100644 --- a/sdk/examples/reconfigure_basin.rs +++ b/sdk/examples/reconfigure_basin.rs @@ -1,8 +1,8 @@ use s2_sdk::{ S2, types::{ - BasinName, BasinReconfiguration, ReconfigureBasinInput, S2Config, StorageClass, - StreamReconfiguration, TimestampingReconfiguration, + BasinName, BasinReconfiguration, ReconfigureBasinInput, S2Config, StreamReconfiguration, + TimestampingReconfiguration, }, }; @@ -22,7 +22,7 @@ async fn main() -> Result<(), Box> { BasinReconfiguration::new() .with_default_stream_config( StreamReconfiguration::new() - .with_storage_class(StorageClass::Standard) + .with_storage_class("standard") .with_timestamping(TimestampingReconfiguration::new().with_uncapped(true)), ) .with_create_stream_on_read(true), diff --git a/sdk/src/api.rs b/sdk/src/api.rs index e518babc..84175479 100644 --- a/sdk/src/api.rs +++ b/sdk/src/api.rs @@ -18,7 +18,10 @@ use s2_api::v1::{ basin::{ BasinInfo, CreateBasinRequest, EnsureBasinRequest, ListBasinsRequest, ListBasinsResponse, }, - config::{BasinConfig, BasinReconfiguration, StreamConfig, StreamReconfiguration}, + config::{ + BasinConfig, BasinReconfiguration, STREAM_CONFIG_HEADER, StreamConfig, + StreamReconfiguration, + }, location::LocationInfo, metrics::{ AccountMetricSetRequest, BasinMetricSetRequest, MetricSetResponse, StreamMetricSetRequest, @@ -43,7 +46,7 @@ use crate::{ error::{ClientError, server_error_has_no_side_effects, server_error_is_retryable}, frame_signal::FrameSignal, reconnect::ReconnectAdvice, - retry::{RetryBackoff, RetryBackoffBuilder}, + retry::{AppendRetryError, RetryBackoff, RetryBackoffBuilder}, types::{ AccessToken, AccessTokenId, AccessTokenMode, AppendRetryPolicy, BasinAuthority, BasinName, Compression, EncryptionKey, LocationName, RetryConfig, S2Config, S2Endpoints, StreamName, @@ -384,6 +387,7 @@ impl BasinClient { name: &StreamName, input: AppendInput, encryption: Option<&EncryptionKey>, + stream_config: Option<&StreamConfig>, append_retry_policy: AppendRetryPolicy, ) -> Result { let url = self.uri(format!("v1/streams/{}/records", urlencoding::encode(name))); @@ -394,6 +398,7 @@ impl BasinClient { .body(input.encode_to_vec()) .build()?; set_encryption_header(&mut request, encryption); + set_stream_config_header(&mut request, stream_config); let response = self .request(request) .with_append_retry_policy(append_retry_policy) @@ -420,6 +425,7 @@ impl BasinClient { start: ReadStart, end: ReadEnd, encryption: Option<&EncryptionKey>, + stream_config: Option<&StreamConfig>, ) -> Result { let url = self.uri(format!("v1/streams/{}/records", urlencoding::encode(name))); let mut builder = self @@ -433,6 +439,7 @@ impl BasinClient { } let mut request = builder.build()?; set_encryption_header(&mut request, encryption); + set_stream_config_header(&mut request, stream_config); let response = self .request(request) .error_handler(read_response_error_handler) @@ -446,6 +453,7 @@ impl BasinClient { name: &StreamName, inputs: I, encryption: Option<&EncryptionKey>, + stream_config: Option<&StreamConfig>, frame_signal: Option, reconnect: ReconnectAdvice, ) -> Result, ApiError> @@ -476,6 +484,7 @@ impl BasinClient { add_basin_header_if_required(request_builder, &self.config.endpoints, &self.name); let mut request = request_builder.build()?; set_encryption_header(&mut request, encryption); + set_stream_config_header(&mut request, stream_config); let (response, access_token) = self.client.init_streaming_authorized(request).await?; let response = match response.into_result().await { Ok(response) => response, @@ -538,6 +547,7 @@ impl BasinClient { start: ReadStart, end: ReadEnd, encryption: Option<&EncryptionKey>, + stream_config: Option<&StreamConfig>, reconnect: ReconnectAdvice, ) -> Result, ApiError> { let url = self.uri(format!("v1/streams/{}/records", urlencoding::encode(name))); @@ -553,6 +563,7 @@ impl BasinClient { add_basin_header_if_required(request_builder, &self.config.endpoints, &self.name); let mut request = request_builder.build()?; set_encryption_header(&mut request, encryption); + set_stream_config_header(&mut request, stream_config); let (response, access_token) = self.client.init_streaming_authorized(request).await?; let response = match response.into_result().await { Ok(response) => response, @@ -656,6 +667,13 @@ pub(crate) enum ApiError { Compression(#[from] std::io::Error), #[error("append condition check failed")] AppendConditionFailed(AppendConditionFailed), + #[error( + "append may have taken effect in an earlier attempt; final attempt failed: {final_attempt_error}" + )] + IndefiniteFailure { + #[source] + final_attempt_error: Box, + }, #[error("read from an unwritten position")] ReadUnwritten(TailResponse), #[error("{1}")] @@ -667,6 +685,9 @@ impl ApiError { match self { Self::Server(status, err_resp) => server_error_is_retryable(*status, &err_resp.code), Self::Client(err) => err.is_retryable(), + Self::IndefiniteFailure { + final_attempt_error, + } => final_attempt_error.is_retryable(), #[cfg(feature = "_hidden")] Self::AccessTokenProvider(error) => error.is_retryable(), _ => false, @@ -687,18 +708,27 @@ impl ApiError { Self::Server(StatusCode::UNAUTHORIZED, response) if response.code == "authn" ) } +} - pub fn has_no_side_effects(&self) -> bool { +impl AppendRetryError for ApiError { + fn has_no_side_effects(&self) -> bool { match self { Self::Server(status, err_resp) => { server_error_has_no_side_effects(*status, &err_resp.code) } Self::Client(err) => err.has_no_side_effects(), + Self::AppendConditionFailed(_) | Self::MalformedAccessToken(_) => true, #[cfg(feature = "_hidden")] Self::AccessTokenProvider(_) => true, _ => false, } } + + fn into_indefinite_failure(self) -> Self { + Self::IndefiniteFailure { + final_attempt_error: Box::new(self), + } + } } impl From for ApiError { @@ -794,7 +824,10 @@ impl BaseClient { C: client::Connect + Clone + Send + Sync + 'static, { let access_token_mode = config.access_token.mode(); - let mut default_headers = HeaderMap::new(); + let mut default_headers = config.default_headers.clone(); + // Authorization belongs to the configured token, including when a + // refreshable provider supplies it immediately before each attempt. + default_headers.remove(AUTHORIZATION); #[cfg(feature = "_hidden")] let mut access_token_provider = None; match &config.access_token { @@ -965,6 +998,14 @@ fn set_encryption_header(request: &mut client::Request, encryption: Option<&Encr } } +fn set_stream_config_header(request: &mut client::Request, stream_config: Option<&StreamConfig>) { + if let Some(config) = stream_config { + request + .headers_mut() + .insert(STREAM_CONFIG_HEADER.clone(), config.to_header_value()); + } +} + pub fn retry_builder(config: &RetryConfig) -> RetryBackoffBuilder { RetryBackoffBuilder::default() .with_min_base_delay(config.min_base_delay) @@ -1013,6 +1054,7 @@ impl<'a> RequestBuilder<'a> { let mut retry_backoff: Option = self .retry_enabled .then(|| self.client.retry_builder.build()); + let mut prior_uncertainty = false; loop { if let Some(ref signal) = self.frame_signal { @@ -1090,6 +1132,11 @@ impl<'a> RequestBuilder<'a> { num_retries_remaining = retry_backoff.as_ref().map(|b| b.remaining()).unwrap_or(0), "retrying request" ); + if self.append_retry_policy.is_some() + && err.attempt_may_have_side_effects(self.frame_signal.as_ref()) + { + prior_uncertainty = true; + } tokio::time::sleep(backoff).await; } else { debug!( @@ -1099,7 +1146,7 @@ impl<'a> RequestBuilder<'a> { retries_exhausted = retry_backoff.as_ref().is_none_or(|b| b.is_exhausted()), "not retrying request" ); - return Err(err); + return Err(err.with_prior_uncertainty(prior_uncertainty)); } } } @@ -1113,9 +1160,7 @@ fn is_safe_to_retry( ) -> bool { let policy_compliant = match policy { None | Some(AppendRetryPolicy::All) => true, - Some(AppendRetryPolicy::NoSideEffects) => { - !frame_signal.is_none_or(|s| s.is_signalled()) || err.has_no_side_effects() - } + Some(AppendRetryPolicy::NoSideEffects) => !err.attempt_may_have_side_effects(frame_signal), }; policy_compliant && (err.is_retryable() @@ -1261,11 +1306,11 @@ fn provision_result_from_parts( #[cfg(test)] mod tests { + use std::error::Error; #[cfg(feature = "_hidden")] - use std::sync::{ - Mutex, - atomic::{AtomicBool, AtomicUsize, Ordering}, - }; + use std::sync::Mutex; + #[cfg(feature = "_hidden")] + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; #[cfg(feature = "_hidden")] use async_trait::async_trait; @@ -1273,6 +1318,243 @@ mod tests { use hyper_util::client::legacy::connect::HttpConnector; use super::*; + use crate::error::AppendError; + + #[cfg(feature = "_hidden")] + #[derive(Default)] + struct HeaderCapture { + unary: Mutex>, + streaming: Mutex>, + } + + #[cfg(feature = "_hidden")] + #[async_trait] + impl client::RequestExecutor for HeaderCapture { + async fn execute_unary( + &self, + mut request: client::Request, + ) -> Result { + let mut headers = self.unary.lock().unwrap(); + headers.push(request.headers_mut().clone()); + // Exercise a real retry through RequestBuilder::send. + Ok(if headers.len() == 1 { + UnaryResponse::new_for_test( + StatusCode::SERVICE_UNAVAILABLE, + br#"{"code":"unavailable","message":"retry"}"#.to_vec(), + ) + } else { + UnaryResponse::new_for_test( + StatusCode::OK, + br#"{"basins":[],"streams":[],"has_more":false}"#.to_vec(), + ) + }) + } + + async fn init_streaming( + &self, + mut request: client::Request, + ) -> Result { + self.streaming + .lock() + .unwrap() + .push(request.headers_mut().clone()); + // Capturing initiation is sufficient: do not construct a response body. + Err(client::HttpError::Timeout) + } + } + + #[cfg(feature = "_hidden")] + #[tokio::test] + async fn default_headers_reach_account_basin_streaming_and_retry_requests() { + let mut session = HeaderValue::from_static("session-1"); + session.set_sensitive(true); + let headers = HeaderMap::from_iter([ + ( + http::header::HeaderName::from_static("x-origin-session"), + session, + ), + ( + AUTHORIZATION, + HeaderValue::from_static("Bearer wrong-token"), + ), + ( + http::header::USER_AGENT, + HeaderValue::from_static("wrong-agent"), + ), + ( + http::header::ACCEPT_ENCODING, + HeaderValue::from_static("wrong-encoding"), + ), + ( + http::header::HeaderName::from_static(S2_BASIN), + HeaderValue::from_static("wrong-basin"), + ), + ]); + let config = S2Config::new("actual-token") + .with_endpoints(S2Endpoints::for_endpoint("http://example.test").unwrap()) + .with_compression(Compression::Gzip) + .with_default_headers(headers) + .unwrap(); + let executor = Arc::new(HeaderCapture::default()); + let mut base = BaseClient::init_with_connector(&config, HttpConnector::new()).unwrap(); + base.client = executor.clone(); + base.retry_builder = RetryBackoffBuilder::default() + .with_min_base_delay(Duration::ZERO) + .with_max_base_delay(Duration::ZERO) + .with_max_retries(1); + let account = AccountClient::init(config.clone(), base); + account + .list_basins(ListBasinsRequest { + prefix: None, + start_after: None, + limit: None, + }) + .await + .unwrap(); + let basin = account.basin_client("test-basin".parse().unwrap()); + basin + .list_streams(ListStreamsRequest { + prefix: None, + start_after: None, + limit: None, + }) + .await + .unwrap(); + + let stream = "test-stream".parse().unwrap(); + assert!( + basin + .read_session( + &stream, + ReadStart { + seq_num: None, + timestamp: None, + tail_offset: None, + clamp: None + }, + ReadEnd { + count: None, + bytes: None, + until: None, + wait: None + }, + None, + None, + ReconnectAdvice::default(), + ) + .await + .is_err() + ); + assert!( + basin + .append_session( + &stream, + futures_util::stream::empty(), + None, + None, + None, + ReconnectAdvice::default(), + ) + .await + .is_err() + ); + + let unary = executor.unary.lock().unwrap(); + let streaming = executor.streaming.lock().unwrap(); + assert_eq!(unary.len(), 3); // account, account retry, basin + assert_eq!(streaming.len(), 2); // read and append + for headers in unary.iter().chain(streaming.iter()) { + assert_eq!(headers["x-origin-session"], "session-1"); + assert!(headers["x-origin-session"].is_sensitive()); + assert_eq!(headers[AUTHORIZATION], "Bearer actual-token"); + assert!(headers[AUTHORIZATION].is_sensitive()); + assert_eq!(headers[http::header::USER_AGENT], config.user_agent); + assert_eq!(headers[http::header::ACCEPT_ENCODING], "gzip"); + assert!(!headers.contains_key(http::header::CONTENT_ENCODING)); + } + assert_eq!(unary[2][S2_BASIN], "test-basin"); + for headers in streaming.iter() { + assert_eq!(headers[S2_BASIN], "test-basin"); + assert_eq!(headers[CONTENT_TYPE], CONTENT_TYPE_S2S); + } + } + + #[cfg(feature = "_hidden")] + #[rstest::rstest] + #[case::none(Compression::None, None, "gzip, zstd")] + #[case::gzip(Compression::Gzip, Some("gzip"), "gzip")] + #[case::zstd(Compression::Zstd, Some("zstd"), "zstd")] + #[tokio::test] + async fn default_accept_encoding_respects_configured_compression( + #[case] compression: Compression, + #[case] content_encoding: Option<&str>, + #[case] accept_encoding: &str, + ) { + let config = S2Config::new("token") + .with_compression(compression) + .with_default_headers(HeaderMap::from_iter([( + http::header::ACCEPT_ENCODING, + HeaderValue::from_static("gzip, zstd"), + )])) + .unwrap(); + let client = BaseClient::init_with_connector(&config, HttpConnector::new()).unwrap(); + let mut request = client + .post("http://example.test/v1/basins".parse().unwrap()) + .json(&serde_json::json!({"name": "test-basin"})) + .build() + .unwrap() + .compress() + .await + .unwrap(); + let headers = request.headers_mut(); + assert_eq!( + headers + .get(http::header::CONTENT_ENCODING) + .map(|value| value.to_str().unwrap()), + content_encoding + ); + assert_eq!(headers[http::header::ACCEPT_ENCODING], accept_encoding); + } + + #[cfg(feature = "_hidden")] + #[tokio::test] + async fn default_headers_are_replaced_and_isolated_between_clients() { + let first_headers = HeaderMap::from_iter([ + ( + http::header::HeaderName::from_static("x-origin-session"), + HeaderValue::from_static("first"), + ), + ( + http::header::HeaderName::from_static("x-first-only"), + HeaderValue::from_static("present"), + ), + ]); + let first = S2Config::new("token") + .with_default_headers(first_headers) + .unwrap(); + let second = first + .clone() + .with_default_headers(HeaderMap::from_iter([( + http::header::HeaderName::from_static("x-origin-session"), + HeaderValue::from_static("second"), + )])) + .unwrap(); + for (config, session) in [(&first, "first"), (&second, "second")] { + let client = BaseClient::init_with_connector(config, HttpConnector::new()).unwrap(); + let mut request = client + .get("http://example.test".parse().unwrap()) + .build() + .unwrap(); + assert_eq!(request.headers_mut()["x-origin-session"], session); + assert_eq!( + request.headers_mut().contains_key("x-first-only"), + session == "first" + ); + } + let cleared = second.with_default_headers(HeaderMap::new()).unwrap(); + assert!(cleared.default_headers.is_empty()); + assert!(S2Config::new("token").default_headers.is_empty()); + } #[cfg(feature = "_hidden")] #[derive(Debug)] @@ -1359,6 +1641,78 @@ mod tests { } } + #[rstest::rstest] + #[case(StatusCode::FORBIDDEN, "permission_denied", false, false)] + #[case(StatusCode::FORBIDDEN, "permission_denied", true, true)] + #[case(StatusCode::SERVICE_UNAVAILABLE, "unavailable", false, false)] + #[case(StatusCode::SERVICE_UNAVAILABLE, "unavailable", true, false)] + #[case(StatusCode::TOO_MANY_REQUESTS, "rate_limited", true, true)] + #[test] + fn unary_append_failure_preserves_uncertainty_and_final_error( + #[case] status: StatusCode, + #[case] code: &str, + #[case] prior_uncertainty: bool, + #[case] wrapped: bool, + ) { + let error = + AppendError::from(server_error(status, code).with_prior_uncertainty(prior_uncertainty)); + assert_eq!( + matches!(error, AppendError::IndefiniteFailure { .. }), + wrapped + ); + let server = error.request_error().unwrap().server_error().unwrap(); + assert_eq!((server.status, server.code.as_str()), (status, code)); + assert_eq!(server.message, "test"); + assert_eq!(error.is_retryable(), server.is_retryable()); + assert_eq!( + error.has_no_side_effects(), + !wrapped && server.has_no_side_effects() + ); + if wrapped { + let latest = error + .source() + .unwrap() + .downcast_ref::>() + .unwrap(); + assert!(latest.has_no_side_effects()); + assert_eq!( + latest.request_error().unwrap().server_error().unwrap().code, + code + ); + assert_eq!( + error.to_string(), + format!( + "append may have taken effect in an earlier attempt; final attempt failed: {latest}" + ) + ); + } + } + + #[rstest::rstest] + #[case::condition_failed( + ApiError::AppendConditionFailed(AppendConditionFailed::SeqNumMismatch(42)), + "sequence number mismatch, expected: 42" + )] + #[case::malformed_access_token( + ApiError::MalformedAccessToken("invalid header".to_owned()), + "malformed access token: invalid header" + )] + #[test] + fn unary_append_wraps_definite_terminal_errors(#[case] error: ApiError, #[case] message: &str) { + assert!(error.has_no_side_effects()); + let error = AppendError::from(error.with_prior_uncertainty(true)); + assert!(matches!(error, AppendError::IndefiniteFailure { .. })); + assert!(!error.has_no_side_effects()); + assert!(!error.is_retryable()); + let latest = error + .source() + .unwrap() + .downcast_ref::>() + .unwrap(); + assert!(latest.has_no_side_effects()); + assert_eq!(latest.to_string(), message); + } + fn server_error(status: StatusCode, code: &str) -> ApiError { ApiError::Server( status, @@ -1435,9 +1789,15 @@ mod tests { #[cfg(feature = "_hidden")] #[tokio::test] async fn dynamic_access_token_is_loaded_for_each_attempt_and_marked_sensitive() { - let config = S2Config::new("unused").with_access_token_provider(RotatingTokenProvider { - generation: AtomicUsize::new(1), - }); + let config = S2Config::new("unused") + .with_default_headers(HeaderMap::from_iter([( + AUTHORIZATION, + HeaderValue::from_static("Bearer wrong-token"), + )])) + .unwrap() + .with_access_token_provider(RotatingTokenProvider { + generation: AtomicUsize::new(1), + }); let client = BaseClient::init_with_connector(&config, HttpConnector::new()).unwrap(); let uri = "http://example.test/v1/basins".parse().unwrap(); let mut request = client.get(uri).build().unwrap(); diff --git a/sdk/src/batching.rs b/sdk/src/batching.rs index 00086f1f..eac2c5ab 100644 --- a/sdk/src/batching.rs +++ b/sdk/src/batching.rs @@ -174,6 +174,7 @@ impl Stream for AppendInputs { records: batch, match_seq_num, fencing_token: self.fencing_token.clone(), + stream_config: None, }))) } Poll::Ready(Some(Err(err))) => Poll::Ready(Some(Err(err))), diff --git a/sdk/src/client.rs b/sdk/src/client.rs index b38d3f31..abf927ab 100644 --- a/sdk/src/client.rs +++ b/sdk/src/client.rs @@ -304,9 +304,8 @@ impl RequestBuilder { } pub fn headers(mut self, headers: &HeaderMap) -> Self { - for (key, value) in headers { - self.headers.insert(key.clone(), value.clone()); - } + // An owned HeaderMap replaces each key's existing values while preserving duplicates. + self.headers.extend(headers.clone()); self } @@ -574,11 +573,7 @@ fn build_http_request( let mut builder = http::Request::builder().method(method).uri(uri.clone()); if let Some(req_headers) = builder.headers_mut() { - for (key, value) in headers { - if let Some(key) = key { - req_headers.insert(key, value); - } - } + *req_headers = headers; if let Some(encoding) = content_encoding { req_headers.insert(CONTENT_ENCODING, encoding); } @@ -1016,6 +1011,37 @@ mod tests { Pool::new(HttpConnector::new()) } + #[test] + fn default_headers_preserve_multiple_values_and_sensitive_flags_on_wire_request() { + let mut headers = HeaderMap::new(); + headers.append("x-tag", HeaderValue::from_static("first")); + let mut second = HeaderValue::from_static("second"); + second.set_sensitive(true); + headers.append("x-tag", second); + headers.insert(CONTENT_ENCODING, HeaderValue::from_static("wrong-encoding")); + + let request = RequestBuilder::get("http://example.test".parse().unwrap()) + .header("x-tag", "replaced") + .headers(&headers) + .build() + .unwrap(); + let cloned = request.try_clone().unwrap(); + let http = build_http_request( + cloned.method, + &cloned.uri, + cloned.headers, + cloned.body.into_http_body(), + Some(HeaderValue::from_static("gzip")), + ) + .unwrap(); + let values = http.headers().get_all("x-tag").iter().collect::>(); + assert_eq!(values.len(), 2); + assert_eq!(values[0], "first"); + assert_eq!(values[1], "second"); + assert!(values[1].is_sensitive()); + assert_eq!(http.headers()[CONTENT_ENCODING], "gzip"); + } + #[test] fn uri_with_path_percent_encoded_segment() { let base: Uri = "https://example.com".parse().unwrap(); diff --git a/sdk/src/error.rs b/sdk/src/error.rs index 7fd07726..bdeb5e53 100644 --- a/sdk/src/error.rs +++ b/sdk/src/error.rs @@ -332,12 +332,28 @@ pub enum AppendError { /// The append condition did not match. #[error(transparent)] ConditionFailed(#[from] AppendConditionFailed), + /// The final attempt failed definitively, but an earlier attempt may have taken effect, + /// so the entire append operation is indeterminate. + #[error( + "append may have taken effect in an earlier attempt; final attempt failed: {final_attempt_error}" + )] + IndefiniteFailure { + /// The definite error returned by the final attempt. + #[source] + final_attempt_error: Box, + }, } impl AppendError { /// Whether retrying the operation is safe or sensible. pub fn is_retryable(&self) -> bool { - matches!(self, Self::Request(error) if error.is_retryable()) + match self { + Self::Request(error) => error.is_retryable(), + Self::ConditionFailed(_) => false, + Self::IndefiniteFailure { + final_attempt_error, + } => final_attempt_error.is_retryable(), + } } /// Whether retrying the operation cannot duplicate a mutation. @@ -345,6 +361,7 @@ impl AppendError { match self { Self::Request(error) => error.has_no_side_effects(), Self::ConditionFailed(_) => true, + Self::IndefiniteFailure { .. } => false, } } @@ -353,6 +370,9 @@ impl AppendError { match self { Self::Request(error) => Some(error), Self::ConditionFailed(_) => None, + Self::IndefiniteFailure { + final_attempt_error, + } => final_attempt_error.request_error(), } } } @@ -361,6 +381,11 @@ impl From for AppendError { fn from(error: ApiError) -> Self { match error { ApiError::AppendConditionFailed(condition) => Self::ConditionFailed(condition.into()), + ApiError::IndefiniteFailure { + final_attempt_error, + } => Self::IndefiniteFailure { + final_attempt_error: Box::new((*final_attempt_error).into()), + }, other => Self::Request(other.into()), } } diff --git a/sdk/src/ops.rs b/sdk/src/ops.rs index be287bec..db339709 100644 --- a/sdk/src/ops.rs +++ b/sdk/src/ops.rs @@ -4,7 +4,9 @@ use crate::{ api::{AccountClient, BaseClient, BasinClient}, error::{AppendError, ReadError, RequestError}, producer::{Producer, ProducerConfig}, - session::{self, AppendSession, AppendSessionConfig, ReadSession, ReadSessionError}, + session::{ + self, AppendSession, AppendSessionConfig, ReadSession, ReadSessionError, StreamHeaders, + }, types::{ AccessTokenId, AccessTokenInfo, AppendAck, AppendInput, BasinConfig, BasinInfo, BasinName, CreateBasinInput, CreateStreamInput, DeleteBasinInput, DeleteStreamInput, EncryptionKey, @@ -437,6 +439,13 @@ impl S2Stream { } } + fn headers(&self, stream_config: Option<&StreamConfig>) -> StreamHeaders { + StreamHeaders { + encryption: self.encryption.clone(), + stream_config: stream_config.cloned().map(Into::into), + } + } + /// Check tail position. pub async fn check_tail(&self) -> Result { let response = self.client.check_tail(&self.name).await?; @@ -444,13 +453,18 @@ impl S2Stream { } /// Append records. - pub async fn append(&self, input: AppendInput) -> Result { + pub async fn append(&self, mut input: AppendInput) -> Result { + let stream_config = input + .stream_config + .take() + .map(s2_api::v1::config::StreamConfig::from); let ack = self .client .append( &self.name, input.into(), self.encryption.as_ref(), + stream_config.as_ref(), self.client.config.retry.append_retry_policy, ) .await?; @@ -459,6 +473,9 @@ impl S2Stream { /// Read records. pub async fn read(&self, input: ReadInput) -> Result { + let stream_config = input + .stream_config + .map(s2_api::v1::config::StreamConfig::from); let batch = self .client .read( @@ -466,6 +483,7 @@ impl S2Stream { input.start.into(), input.stop.into(), self.encryption.as_ref(), + stream_config.as_ref(), ) .await?; let mut batch = ReadBatch::from_api(batch); @@ -480,7 +498,7 @@ impl S2Stream { AppendSession::new( self.client.clone(), self.name.clone(), - self.encryption.clone(), + self.headers(config.stream_config()), config, ) } @@ -490,7 +508,7 @@ impl S2Stream { Producer::new( self.client.clone(), self.name.clone(), - self.encryption.clone(), + self.headers(config.stream_config()), config, ) } @@ -504,7 +522,7 @@ impl S2Stream { session::read_session( self.client.clone(), self.name.clone(), - self.encryption.clone(), + self.headers(input.stream_config.as_ref()), input, config, ) diff --git a/sdk/src/producer.rs b/sdk/src/producer.rs index 632c31b5..293f774a 100644 --- a/sdk/src/producer.rs +++ b/sdk/src/producer.rs @@ -20,9 +20,11 @@ use crate::{ api::BasinClient, batching::{AppendInputs, AppendRecordBatches, BatchingConfig}, error::ProducerError, - session::{AppendPermit, AppendPermits, AppendSessionInternal, BatchSubmitTicket}, + session::{ + AppendPermit, AppendPermits, AppendSessionInternal, BatchSubmitTicket, StreamHeaders, + }, types::{ - AppendAck, AppendRecord, EncryptionKey, FencingToken, MeteredBytes, ONE_MIB, StreamName, + AppendAck, AppendRecord, FencingToken, MeteredBytes, ONE_MIB, StreamConfig, StreamName, ValidationError, }, }; @@ -75,6 +77,7 @@ pub struct ProducerConfig { batching: BatchingConfig, fencing_token: Option, match_seq_num: Option, + stream_config: Option, } impl Default for ProducerConfig { @@ -84,6 +87,7 @@ impl Default for ProducerConfig { batching: BatchingConfig::default(), fencing_token: None, match_seq_num: None, + stream_config: None, } } } @@ -137,6 +141,23 @@ impl ProducerConfig { ..self } } + + /// Set the stream configuration to apply if the stream is created on append. + /// + /// Unset fields inherit the basin's default stream configuration. Ignored if the stream + /// already exists. + /// + /// Defaults to `None`. + pub fn with_stream_config(self, stream_config: StreamConfig) -> Self { + Self { + stream_config: Some(stream_config), + ..self + } + } + + pub(crate) fn stream_config(&self) -> Option<&StreamConfig> { + self.stream_config.as_ref() + } } /// High-level interface for submitting individual [`AppendRecord`]s. @@ -154,12 +175,12 @@ impl Producer { pub(crate) fn new( client: BasinClient, stream: StreamName, - encryption: Option, + headers: StreamHeaders, config: ProducerConfig, ) -> Self { let (cmd_tx, cmd_rx) = mpsc::channel::(RECORD_BATCH_MAX.count); let permits = AppendPermits::new(None, config.max_unacked_bytes); - let session = AppendSessionInternal::new(client, stream, encryption); + let session = AppendSessionInternal::new(client, stream, headers); let terminal_err = Arc::new(OnceLock::new()); let _handle = AbortOnDropHandle::new(tokio::spawn(Self::run( session, diff --git a/sdk/src/retry.rs b/sdk/src/retry.rs index ac7bb370..8eddd738 100644 --- a/sdk/src/retry.rs +++ b/sdk/src/retry.rs @@ -2,6 +2,33 @@ use std::time::Duration; use rand::{RngExt, rng}; +use crate::frame_signal::FrameSignal; + +pub(crate) trait AppendRetryError: Sized { + /// Whether the failure represented by this error guarantees that no mutation occurred. + fn has_no_side_effects(&self) -> bool; + + /// Mark the whole append as indefinite, retaining this final attempt's definite error. + /// The caller must establish that an earlier attempt may have taken effect. + fn into_indefinite_failure(self) -> Self; + + /// Wrap a definite final error if an earlier attempt may have taken effect. + /// Already indefinite errors are returned unchanged. + fn with_prior_uncertainty(self, prior_uncertainty: bool) -> Self { + if prior_uncertainty && self.has_no_side_effects() { + self.into_indefinite_failure() + } else { + self + } + } + + /// Whether this attempt may have taken effect, accounting for unsent request data. + /// Without a frame signal, assume the request may have been sent. + fn attempt_may_have_side_effects(&self, frame_signal: Option<&FrameSignal>) -> bool { + !self.has_no_side_effects() && frame_signal.is_none_or(|s| s.is_signalled()) + } +} + #[derive(Debug, Clone, Copy)] pub struct RetryBackoffBuilder { pub min_base_delay: Duration, diff --git a/sdk/src/session/append.rs b/sdk/src/session/append.rs index ea1f90fa..43989a7b 100644 --- a/sdk/src/session/append.rs +++ b/sdk/src/session/append.rs @@ -23,10 +23,11 @@ use crate::{ error::{AppendError, RequestError}, frame_signal::FrameSignal, reconnect::{AdvisedReconnects, ReconnectAdvice}, - retry::RetryBackoffBuilder, + retry::{AppendRetryError, RetryBackoffBuilder}, + session::StreamHeaders, types::{ - AccessTokenMode, AppendAck, AppendInput, AppendRetryPolicy, EncryptionKey, MeteredBytes, - ONE_MIB, StreamName, StreamPosition, ValidationError, + AccessTokenMode, AppendAck, AppendInput, AppendRetryPolicy, MeteredBytes, ONE_MIB, + StreamConfig, StreamName, StreamPosition, ValidationError, }, }; @@ -58,6 +59,16 @@ pub enum AppendSessionError { /// The server returned an invalid append acknowledgement. #[error("invalid append acknowledgement: {0}")] InvalidAck(String), + /// The final attempt failed definitively, but an earlier attempt may have taken effect, + /// so the entire append operation is indeterminate. + #[error( + "append may have taken effect in an earlier attempt; final attempt failed: {final_attempt_error}" + )] + IndefiniteFailure { + /// The definite error returned by the final attempt. + #[source] + final_attempt_error: Box, + }, } impl AppendSessionError { @@ -65,6 +76,9 @@ impl AppendSessionError { pub fn is_retryable(&self) -> bool { match self { Self::Append(error) => error.is_retryable(), + Self::IndefiniteFailure { + final_attempt_error, + } => final_attempt_error.is_retryable(), Self::AckTimeout | Self::ServerDisconnected => true, Self::StreamClosedEarly | Self::SessionClosed @@ -78,6 +92,7 @@ impl AppendSessionError { pub fn has_no_side_effects(&self) -> bool { match self { Self::Append(error) => error.has_no_side_effects(), + Self::IndefiniteFailure { .. } => false, Self::SessionClosed | Self::SessionClosing => true, Self::AckTimeout | Self::ServerDisconnected @@ -91,6 +106,9 @@ impl AppendSessionError { pub fn request_error(&self) -> Option<&RequestError> { match self { Self::Append(error) => error.request_error(), + Self::IndefiniteFailure { + final_attempt_error, + } => final_attempt_error.request_error(), Self::AckTimeout | Self::ServerDisconnected | Self::StreamClosedEarly @@ -116,14 +134,21 @@ impl AppendSessionError { } } +impl AppendRetryError for AppendSessionError { + fn has_no_side_effects(&self) -> bool { + Self::has_no_side_effects(self) + } + + fn into_indefinite_failure(self) -> Self { + Self::IndefiniteFailure { + final_attempt_error: Box::new(self), + } + } +} + impl From for AppendSessionError { fn from(error: ApiError) -> Self { - match error { - ApiError::AppendConditionFailed(condition) => { - Self::Append(AppendError::ConditionFailed(condition.into())) - } - other => Self::Append(AppendError::Request(other.into())), - } + Self::Append(error.into()) } } @@ -154,6 +179,7 @@ impl Future for BatchSubmitTicket { pub struct AppendSessionConfig { max_unacked_bytes: u32, max_unacked_batches: Option, + stream_config: Option, } impl Default for AppendSessionConfig { @@ -161,6 +187,7 @@ impl Default for AppendSessionConfig { Self { max_unacked_bytes: 5 * ONE_MIB, max_unacked_batches: None, + stream_config: None, } } } @@ -195,6 +222,23 @@ impl AppendSessionConfig { ..self } } + + /// Set the stream configuration to apply if the stream is created on append. + /// + /// Unset fields inherit the basin's default stream configuration. Ignored if the stream + /// already exists. + /// + /// Defaults to `None`. + pub fn with_stream_config(self, stream_config: StreamConfig) -> Self { + Self { + stream_config: Some(stream_config), + ..self + } + } + + pub(crate) fn stream_config(&self) -> Option<&StreamConfig> { + self.stream_config.as_ref() + } } struct SessionState { @@ -231,7 +275,7 @@ impl AppendSession { pub(crate) fn new( client: BasinClient, stream: StreamName, - encryption: Option, + headers: StreamHeaders, config: AppendSessionConfig, ) -> Self { let buffer_size = config @@ -245,7 +289,7 @@ impl AppendSession { let handle = AbortOnDropHandle::new(tokio::spawn(run_session_with_retry( client, stream, - encryption, + headers, cmd_rx, retry_builder, buffer_size, @@ -356,11 +400,7 @@ pub(crate) struct AppendSessionInternal { } impl AppendSessionInternal { - pub(crate) fn new( - client: BasinClient, - stream: StreamName, - encryption: Option, - ) -> Self { + pub(crate) fn new(client: BasinClient, stream: StreamName, headers: StreamHeaders) -> Self { let buffer_size = DEFAULT_CHANNEL_BUFFER_SIZE; let (cmd_tx, cmd_rx) = mpsc::channel(buffer_size); let retry_builder = retry_builder(&client.config.retry); @@ -368,7 +408,7 @@ impl AppendSessionInternal { let handle = AbortOnDropHandle::new(tokio::spawn(run_session_with_retry( client, stream, - encryption, + headers, cmd_rx, retry_builder, buffer_size, @@ -473,7 +513,7 @@ impl AppendPermits { async fn run_session_with_retry( client: BasinClient, stream: StreamName, - encryption: Option, + headers: StreamHeaders, cmd_rx: mpsc::Receiver, retry_builder: RetryBackoffBuilder, buffer_size: usize, @@ -503,7 +543,7 @@ async fn run_session_with_retry( let result = run_session( &client, &stream, - encryption.as_ref(), + &headers, &mut state, buffer_size, &frame_signal, @@ -548,6 +588,11 @@ async fn run_session_with_retry( access_token_mode, ) && let Some(backoff) = retry_backoff.next() { + if err.attempt_may_have_side_effects(frame_signal.as_ref()) { + for append in &mut state.inflight_appends { + append.prior_uncertainty = true; + } + } debug!( %err, ?backoff, @@ -562,12 +607,16 @@ async fn run_session_with_retry( "not retrying append session" ); - let err: AppendSessionError = err; - - let _ = terminal_err.set(err.clone()); + let session_err = err.clone().with_prior_uncertainty( + state.inflight_appends.iter().any(|a| a.prior_uncertainty), + ); + let _ = terminal_err.set(session_err.clone()); for inflight_append in state.inflight_appends.drain(..) { - let _ = inflight_append.ack_tx.send(Err(err.clone())); + let error = err + .clone() + .with_prior_uncertainty(inflight_append.prior_uncertainty); + let _ = inflight_append.ack_tx.send(Err(error)); } if let Some(stashed) = state.stashed_submission.take() { @@ -575,12 +624,16 @@ async fn run_session_with_retry( } if let Some(done_tx) = state.close_tx.take() { - let _ = done_tx.send(Err(err.clone())); + let _ = done_tx.send(Err(session_err.clone())); } state.cmd_rx.close(); while let Some(cmd) = state.cmd_rx.recv().await { - cmd.reject(err.clone()); + let error = match &cmd { + Command::Submit { .. } => &err, + Command::Close { .. } => &session_err, + }; + cmd.reject(error.clone()); } break; } @@ -604,7 +657,7 @@ enum SessionOutcome { async fn run_session( client: &BasinClient, stream: &StreamName, - encryption: Option<&EncryptionKey>, + headers: &StreamHeaders, state: &mut SessionState, buffer_size: usize, frame_signal: &Option, @@ -618,7 +671,7 @@ async fn run_session( let (input_tx, mut acks) = connect( client, stream, - encryption, + headers, buffer_size, frame_signal.clone(), reconnect.clone(), @@ -688,6 +741,7 @@ async fn run_session( ack_tx: submission.ack_tx, ack_deadline, _permit: submission.permit, + prior_uncertainty: false, }); } @@ -888,7 +942,7 @@ async fn drain_for_reconnect( async fn connect( client: &BasinClient, stream: &StreamName, - encryption: Option<&EncryptionKey>, + headers: &StreamHeaders, buffer_size: usize, frame_signal: Option, reconnect: ReconnectAdvice, @@ -899,7 +953,8 @@ async fn connect( .append_session( stream, ReceiverStream::new(input_rx).map(|i| i.into()), - encryption, + headers.encryption.as_ref(), + headers.stream_config.as_ref(), frame_signal, reconnect, ) @@ -982,6 +1037,7 @@ struct InflightAppend { ack_tx: oneshot::Sender>, ack_deadline: Instant, _permit: Option, + prior_uncertainty: bool, } enum Command { @@ -1018,9 +1074,7 @@ fn is_safe_to_retry( let policy_compliant = match policy { AppendRetryPolicy::All => true, AppendRetryPolicy::NoSideEffects => { - !has_inflight - || !frame_signal.is_none_or(|s| s.is_signalled()) - || err.has_no_side_effects() + !has_inflight || !err.attempt_may_have_side_effects(frame_signal) } }; policy_compliant @@ -1056,13 +1110,16 @@ impl From for TimerEvent { #[cfg(test)] mod tests { + use std::error::Error; + use http::StatusCode; use super::{AppendSessionError, is_safe_to_retry}; use crate::{ api::{ApiError, ServerErrorBody}, - error::{AppendError, RequestError}, + error::{AppendError, ProducerError, RequestError}, frame_signal::FrameSignal, + retry::AppendRetryError, types::{AccessTokenMode, AppendRetryPolicy}, }; @@ -1076,6 +1133,61 @@ mod tests { )))) } + #[rstest::rstest] + #[case(StatusCode::FORBIDDEN, "permission_denied", false, false)] + #[case(StatusCode::FORBIDDEN, "permission_denied", true, true)] + #[case(StatusCode::SERVICE_UNAVAILABLE, "unavailable", false, false)] + #[case(StatusCode::SERVICE_UNAVAILABLE, "unavailable", true, false)] + #[case(StatusCode::TOO_MANY_REQUESTS, "rate_limited", true, true)] + #[test] + fn session_failure_preserves_uncertainty_and_latest_error( + #[case] status: StatusCode, + #[case] code: &str, + #[case] prior_uncertainty: bool, + #[case] wrapped: bool, + ) { + let latest = server_error(status, code); + let error = latest.clone().with_prior_uncertainty(prior_uncertainty); + assert_eq!( + matches!(error, AppendSessionError::IndefiniteFailure { .. }), + wrapped + ); + assert_eq!(error.is_retryable(), latest.is_retryable()); + assert_eq!( + error.has_no_side_effects(), + !wrapped && latest.has_no_side_effects() + ); + assert_eq!( + error.request_error().unwrap().server_error().unwrap().code, + code + ); + if wrapped { + let source = error + .source() + .unwrap() + .downcast_ref::>() + .unwrap(); + assert_eq!(source.to_string(), latest.to_string()); + assert!(source.has_no_side_effects()); + } + + let producer_error = ProducerError::from(error.clone()); + assert_eq!(producer_error.is_retryable(), error.is_retryable()); + assert_eq!( + producer_error.has_no_side_effects(), + error.has_no_side_effects() + ); + assert_eq!( + producer_error + .request_error() + .unwrap() + .server_error() + .unwrap() + .code, + code + ); + } + #[test] fn safe_to_retry_session_all_policy() { let retryable = server_error(StatusCode::INTERNAL_SERVER_ERROR, "internal"); diff --git a/sdk/src/session/mod.rs b/sdk/src/session/mod.rs index 107df6e4..7868ed29 100644 --- a/sdk/src/session/mod.rs +++ b/sdk/src/session/mod.rs @@ -5,3 +5,12 @@ pub(crate) use append::{AppendPermit, AppendPermits, AppendSessionInternal, Batc pub use append::{AppendSession, AppendSessionConfig}; pub(crate) use read::read_session; pub use read::{ReadSession, ReadSessionError}; + +/// Per-stream options sent as request headers on every (re)connect of a session. +#[derive(Debug, Clone, Default)] +pub(crate) struct StreamHeaders { + /// `s2-encryption-key` + pub encryption: Option, + /// `s2-stream-config` + pub stream_config: Option, +} diff --git a/sdk/src/session/read.rs b/sdk/src/session/read.rs index a3a07e99..83f848f4 100644 --- a/sdk/src/session/read.rs +++ b/sdk/src/session/read.rs @@ -22,8 +22,9 @@ use crate::{ error::{ReadError, RequestError}, reconnect::{AdvisedReconnects, ReconnectAdvice}, retry::RetryBackoff, + session::StreamHeaders, types::{ - AccessTokenMode, EncryptionKey, MeteredBytes, ReadBatch, ReadInput, ReadSessionConfig, + AccessTokenMode, MeteredBytes, ReadBatch, ReadInput, ReadSessionConfig, ReadSessionRetryPolicy, StreamName, StreamPosition, }, }; @@ -371,7 +372,7 @@ impl Drop for ReadSession { pub async fn read_session( client: BasinClient, name: StreamName, - encryption: Option, + headers: StreamHeaders, input: ReadInput, config: ReadSessionConfig, ) -> Result { @@ -379,6 +380,7 @@ pub async fn read_session( start, stop, ignore_command_records, + stream_config: _, } = input; let mut start: ReadStart = start.into(); let mut end: ReadEnd = stop.into(); @@ -399,7 +401,7 @@ pub async fn read_session( match session_inner( client.clone(), name.clone(), - encryption.clone(), + headers.clone(), start.clone(), end.clone(), ReconnectAdvice::default(), @@ -439,7 +441,7 @@ pub async fn read_session( match session_inner( client.clone(), name.clone(), - encryption.clone(), + headers.clone(), start.clone(), end.clone(), ReconnectAdvice::default(), @@ -579,14 +581,21 @@ fn update_resume_start(start: &mut ReadStart, batch: &ReadBatch) { async fn session_inner( client: BasinClient, name: StreamName, - encryption: Option, + headers: StreamHeaders, start: ReadStart, end: ReadEnd, reconnect: ReconnectAdvice, advised_reconnects: AdvisedReconnects, ) -> Result, ReadSessionFailure> { let mut batches = client - .read_session(&name, start, end, encryption.as_ref(), reconnect.clone()) + .read_session( + &name, + start, + end, + headers.encryption.as_ref(), + headers.stream_config.as_ref(), + reconnect.clone(), + ) .await?; let mut declined_advice = false; diff --git a/sdk/src/types.rs b/sdk/src/types.rs index 6d324782..099048d2 100644 --- a/sdk/src/types.rs +++ b/sdk/src/types.rs @@ -15,7 +15,9 @@ use std::{ #[cfg(feature = "_hidden")] use async_trait::async_trait; use bytes::Bytes; +use compact_str::CompactString; use http::{ + HeaderMap, header::HeaderValue, uri::{Authority, Scheme}, }; @@ -511,6 +513,7 @@ pub struct S2Config { pub(crate) retry: RetryConfig, pub(crate) compression: Compression, pub(crate) user_agent: HeaderValue, + pub(crate) default_headers: HeaderMap, pub(crate) insecure_skip_cert_verification: bool, pub(crate) rustls_crypto_provider: Option>, } @@ -528,6 +531,7 @@ impl S2Config { user_agent: concat!("s2-sdk-rust/", env!("CARGO_PKG_VERSION")) .parse() .expect("valid user agent"), + default_headers: HeaderMap::new(), insecure_skip_cert_verification: false, rustls_crypto_provider: default_rustls_crypto_provider(), } @@ -547,6 +551,54 @@ impl S2Config { Self { endpoints, ..self } } + /// Set additional HTTP headers to send with every request. + /// + /// These headers apply to account, basin, and stream operations, including + /// retries and streaming requests. SDK-generated headers, such as + /// authorization and basin routing, take precedence over these defaults. + /// Calling this method again replaces the previous set of default headers. + /// + /// `Accept-Encoding` defaults are used only when [`Compression::None`] is + /// configured; otherwise the SDK sets the header to the configured + /// compression algorithm. + /// + /// Headers are sent to all configured S2 endpoints. Use + /// [`HeaderValue::set_sensitive`] for values that should be redacted in debug + /// output. Do not use these defaults for per-request identifiers, since the + /// same values are reused across requests. + /// + /// # Errors + /// + /// Returns an error if `default_headers` contains `Content-Type`, + /// `Content-Encoding`, `Content-Length`, or `Transfer-Encoding`. + /// The SDK controls request format and body framing. + /// Use [`Self::with_compression`] to configure request body encoding. + #[cfg(feature = "_hidden")] + #[doc(hidden)] + pub fn with_default_headers(self, default_headers: HeaderMap) -> Result { + if default_headers.contains_key(http::header::CONTENT_ENCODING) { + return Err(ValidationError( + "Content-Encoding cannot be set in default headers; use S2Config::with_compression instead" + .into(), + )); + } + for name in [ + http::header::CONTENT_TYPE, + http::header::CONTENT_LENGTH, + http::header::TRANSFER_ENCODING, + ] { + if default_headers.contains_key(&name) { + return Err(ValidationError(format!( + "{name} cannot be set in default headers; the SDK controls request format and body framing" + ))); + } + } + Ok(Self { + default_headers, + ..self + }) + } + /// Set the timeout for establishing a connection to the server. /// /// Defaults to `3s`. @@ -682,33 +734,6 @@ impl Page { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -/// Storage class for recent appends. -pub enum StorageClass { - /// Standard storage class that offers append latencies under `500ms`. - Standard, - /// Express storage class that offers append latencies under `50ms`. - Express, -} - -impl From for StorageClass { - fn from(value: api::config::StorageClass) -> Self { - match value { - api::config::StorageClass::Standard => StorageClass::Standard, - api::config::StorageClass::Express => StorageClass::Express, - } - } -} - -impl From for api::config::StorageClass { - fn from(value: StorageClass) -> Self { - match value { - StorageClass::Standard => api::config::StorageClass::Standard, - StorageClass::Express => api::config::StorageClass::Express, - } - } -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] /// Retention policy for records in a stream. pub enum RetentionPolicy { @@ -868,10 +893,8 @@ impl From for api::config::DeleteOnEmptyConfig { #[non_exhaustive] /// Configuration for a stream. pub struct StreamConfig { - /// Storage class for the stream. - /// - /// Defaults to [`Express`](StorageClass::Express). - pub storage_class: Option, + /// [Storage class](https://s2.dev/docs/storage-classes) for the stream. + pub storage_class: Option, /// Retention policy for records in the stream. /// /// Defaults to `7 days` of retention. @@ -893,9 +916,9 @@ impl StreamConfig { } /// Set the storage class for the stream. - pub fn with_storage_class(self, storage_class: StorageClass) -> Self { + pub fn with_storage_class(self, storage_class: impl Into) -> Self { Self { - storage_class: Some(storage_class), + storage_class: Some(storage_class.into()), ..self } } @@ -928,7 +951,7 @@ impl StreamConfig { impl From for StreamConfig { fn from(value: api::config::StreamConfig) -> Self { Self { - storage_class: value.storage_class.map(Into::into), + storage_class: value.storage_class, retention_policy: value.retention_policy.map(Into::into), timestamping: value.timestamping.map(Into::into), delete_on_empty: value.delete_on_empty.map(Into::into), @@ -939,7 +962,7 @@ impl From for StreamConfig { impl From for api::config::StreamConfig { fn from(value: StreamConfig) -> Self { Self { - storage_class: value.storage_class.map(Into::into), + storage_class: value.storage_class, retention_policy: value.retention_policy.map(Into::into), timestamping: value.timestamping.map(Into::into), delete_on_empty: value.delete_on_empty.map(Into::into), @@ -1417,7 +1440,7 @@ impl From for api::config::DeleteOnEmptyReconfigur /// Reconfiguration for [`StreamConfig`]. pub struct StreamReconfiguration { /// Override for the existing [`storage_class`](StreamConfig::storage_class). - pub storage_class: Maybe>, + pub storage_class: Maybe>, /// Override for the existing [`retention_policy`](StreamConfig::retention_policy). pub retention_policy: Maybe>, /// Override for the existing [`timestamping`](StreamConfig::timestamping). @@ -1433,9 +1456,9 @@ impl StreamReconfiguration { } /// Set the override for the existing [`storage_class`](StreamConfig::storage_class). - pub fn with_storage_class(self, storage_class: StorageClass) -> Self { + pub fn with_storage_class(self, storage_class: impl Into) -> Self { Self { - storage_class: Maybe::Specified(Some(storage_class)), + storage_class: Maybe::Specified(Some(storage_class.into())), ..self } } @@ -1468,7 +1491,7 @@ impl StreamReconfiguration { impl From for api::config::StreamReconfiguration { fn from(value: StreamReconfiguration) -> Self { Self { - storage_class: value.storage_class.map(|m| m.map(Into::into)), + storage_class: value.storage_class, retention_policy: value.retention_policy.map(|m| m.map(Into::into)), timestamping: value.timestamping.map(|m| m.map(Into::into)), delete_on_empty: value.delete_on_empty.map(|m| m.map(Into::into)), @@ -1660,6 +1683,10 @@ pub struct LocationInfo { pub name: LocationName, /// Location represents a private placement, limited by account. pub is_private: bool, + /// [Storage classes](https://s2.dev/docs/storage-classes) available to the account in this location. + pub storage_classes: Option>, + /// Default [storage class](https://s2.dev/docs/storage-classes) for this location. + pub default_storage_class: Option, } impl From for LocationInfo { @@ -1667,6 +1694,8 @@ impl From for LocationInfo { Self { name: value.name, is_private: value.is_private, + storage_classes: value.storage_classes, + default_storage_class: value.default_storage_class, } } } @@ -2404,7 +2433,7 @@ pub enum BasinMetricSet { /// Returns a [`GaugeMetric`] representing a timeseries of total stored bytes across all streams /// in the basin, with one observed value for each hour over the requested time range. Storage(TimeRange), - /// Returns [`AccumulationMetric`]s, one per storage class (standard, express). + /// Returns [`AccumulationMetric`]s, one per storage class. /// /// Each metric represents a timeseries of the number of append operations across all streams /// in the basin, with one accumulated value per interval over the requested time range. @@ -3362,6 +3391,16 @@ pub struct AppendInput { /// If unspecified, no matching is performed. If specified and mismatched, /// the append fails. A stream defaults to `""` as its fencing token. pub fencing_token: Option, + /// Stream configuration to apply if the stream is created on append. + /// + /// Unset fields inherit the basin's default stream configuration. Ignored if the stream + /// already exists. + /// + /// Only used by [`append`](crate::S2Stream::append). Append sessions send the header once + /// at connect; see + /// [`AppendSessionConfig::with_stream_config`](crate::append_session::AppendSessionConfig::with_stream_config) + /// and [`ProducerConfig::with_stream_config`](crate::producer::ProducerConfig::with_stream_config). + pub stream_config: Option, } impl AppendInput { @@ -3371,6 +3410,15 @@ impl AppendInput { records, match_seq_num: None, fencing_token: None, + stream_config: None, + } + } + + /// Set the stream configuration to apply if the stream is created on append. + pub fn with_stream_config(self, stream_config: StreamConfig) -> Self { + Self { + stream_config: Some(stream_config), + ..self } } @@ -3627,6 +3675,11 @@ pub struct ReadInput { /// /// Defaults to `false`. pub ignore_command_records: bool, + /// Stream configuration to apply if the stream is created on read. + /// + /// Unset fields inherit the basin's default stream configuration. Ignored if the stream + /// already exists. + pub stream_config: Option, } #[derive(Debug, Clone, Copy, Default, Eq, PartialEq)] @@ -3694,6 +3747,14 @@ impl ReadInput { ..self } } + + /// Set the stream configuration to apply if the stream is created on read. + pub fn with_stream_config(self, stream_config: StreamConfig) -> Self { + Self { + stream_config: Some(stream_config), + ..self + } + } } #[derive(Debug, Clone)] @@ -3989,15 +4050,51 @@ mod tests { assert!(!cfg.insecure_skip_cert_verification); } - // -- StorageClass -- - + #[cfg(feature = "_hidden")] #[rstest] - #[case::standard(StorageClass::Standard)] - #[case::express(StorageClass::Express)] - fn storage_class_roundtrip(#[case] sdk: StorageClass) { - let api: api::config::StorageClass = sdk.into(); - let back: StorageClass = api.into(); - assert_eq!(back, sdk); + #[case::matching_compression("content-encoding", "gzip", Compression::Gzip)] + #[case::mixed_case("Content-Encoding", "identity", Compression::None)] + #[case::empty_value("content-encoding", "", Compression::None)] + fn default_headers_reject_content_encoding( + #[case] name: &str, + #[case] value: &str, + #[case] compression: Compression, + ) { + let headers = HeaderMap::from_iter([( + name.parse::().unwrap(), + HeaderValue::from_str(value).unwrap(), + )]); + let error = S2Config::new("token") + .with_compression(compression) + .with_default_headers(headers) + .unwrap_err(); + assert!(error.0.contains("Content-Encoding")); + assert!(error.0.contains("with_compression")); + } + + #[cfg(feature = "_hidden")] + #[rstest] + #[case::content_type_s2s("content-type", "s2s/proto")] + #[case::content_type_protobuf("content-type", "application/protobuf")] + #[case::content_type_json("content-type", "application/json")] + #[case::content_type_mixed_case("Content-Type", "s2s/proto")] + #[case::content_type_empty("content-type", "")] + #[case::content_length("content-length", "123")] + #[case::content_length_mixed_case("Content-Length", "0")] + #[case::content_length_empty("content-length", "")] + #[case::transfer_encoding("transfer-encoding", "chunked")] + #[case::transfer_encoding_mixed_case("Transfer-Encoding", "chunked")] + #[case::transfer_encoding_empty("transfer-encoding", "")] + fn default_headers_reject_framing_headers(#[case] name: &str, #[case] value: &str) { + let headers = HeaderMap::from_iter([( + name.parse::().unwrap(), + HeaderValue::from_str(value).unwrap(), + )]); + let error = S2Config::new("token") + .with_default_headers(headers) + .unwrap_err(); + assert!(error.0.contains(&name.to_ascii_lowercase())); + assert!(error.0.contains("framing")); } // -- RetentionPolicy -- @@ -4061,7 +4158,7 @@ mod tests { #[test] fn stream_config_builder_and_roundtrip() { let sdk = StreamConfig::new() - .with_storage_class(StorageClass::Express) + .with_storage_class("express") .with_retention_policy(RetentionPolicy::Age(86400)) .with_timestamping(TimestampingConfig { mode: Some(TimestampingMode::ClientPrefer), @@ -4078,9 +4175,7 @@ mod tests { #[test] fn basin_config_builder_and_roundtrip() { let sdk = BasinConfig::new() - .with_default_stream_config( - StreamConfig::new().with_storage_class(StorageClass::Standard), - ) + .with_default_stream_config(StreamConfig::new().with_storage_class("standard")) .with_create_stream_on_append(true) .with_create_stream_on_read(false); let api: api::config::BasinConfig = sdk.clone().into(); diff --git a/sdk/tests/account_ops.rs b/sdk/tests/account_ops.rs index 552445d7..4576a36a 100644 --- a/sdk/tests/account_ops.rs +++ b/sdk/tests/account_ops.rs @@ -70,7 +70,7 @@ async fn basin_config_roundtrip() -> Result<(), Box> { let config = BasinConfig::new() .with_default_stream_config( StreamConfig::new() - .with_storage_class(StorageClass::Express) + .with_storage_class("express") .with_delete_on_empty( DeleteOnEmptyConfig::new().with_min_age(Duration::from_secs(60)), ), @@ -85,7 +85,7 @@ async fn basin_config_roundtrip() -> Result<(), Box> { retrieved_config, BasinConfig { default_stream_config: Some(StreamConfig { - storage_class: Some(StorageClass::Express), + storage_class: Some(ref storage_class), delete_on_empty: Some(DeleteOnEmptyConfig { min_age_secs: 60, .. @@ -94,7 +94,7 @@ async fn basin_config_roundtrip() -> Result<(), Box> { }), create_stream_on_read: true, .. - } + } if storage_class == "express" ); s2.delete_basin(DeleteBasinInput::new(basin_name)).await?; @@ -114,9 +114,7 @@ async fn reconfigure_basin() -> Result<(), Box> { .await?; let new_config = BasinReconfiguration::new() - .with_default_stream_config( - StreamReconfiguration::new().with_storage_class(StorageClass::Standard), - ) + .with_default_stream_config(StreamReconfiguration::new().with_storage_class("standard")) .with_create_stream_on_append(false); let updated_config = s2 @@ -127,12 +125,12 @@ async fn reconfigure_basin() -> Result<(), Box> { updated_config, BasinConfig { default_stream_config: Some(StreamConfig { - storage_class: Some(StorageClass::Standard), + storage_class: Some(ref storage_class), .. }), create_stream_on_append: false, .. - } + } if storage_class == "standard" ); s2.delete_basin(DeleteBasinInput::new(basin_name)).await?; @@ -339,7 +337,7 @@ async fn get_basin_config() -> Result<(), Box> { let basin_name = unique_basin_name(); let config = BasinConfig::new() - .with_default_stream_config(StreamConfig::new().with_storage_class(StorageClass::Express)); + .with_default_stream_config(StreamConfig::new().with_storage_class("express")); s2.create_basin(CreateBasinInput::new(basin_name.clone()).with_config(config)) .await?; @@ -349,9 +347,9 @@ async fn get_basin_config() -> Result<(), Box> { assert_matches!( retrieved_config.default_stream_config, Some(StreamConfig { - storage_class: Some(StorageClass::Express), + storage_class: Some(ref storage_class), .. - }) + }) if storage_class == "express" ); s2.delete_basin(DeleteBasinInput::new(basin_name)).await?; diff --git a/sdk/tests/basin_ops.rs b/sdk/tests/basin_ops.rs index 39506e6b..fc9b2cc4 100644 --- a/sdk/tests/basin_ops.rs +++ b/sdk/tests/basin_ops.rs @@ -55,7 +55,7 @@ async fn create_list_and_delete_stream(basin: &S2Basin) -> Result<(), RequestErr async fn stream_config_roundtrip(basin: &S2Basin) -> Result<(), RequestError> { let stream_name = unique_stream_name(); let config = StreamConfig::new() - .with_storage_class(StorageClass::Standard) + .with_storage_class("standard") .with_retention_policy(RetentionPolicy::Age(3600)) .with_timestamping(TimestampingConfig::new().with_mode(TimestampingMode::ClientRequire)); @@ -68,14 +68,14 @@ async fn stream_config_roundtrip(basin: &S2Basin) -> Result<(), RequestError> { assert_matches!( retrieved_config, StreamConfig { - storage_class: Some(StorageClass::Standard), + storage_class: Some(ref storage_class), retention_policy: Some(RetentionPolicy::Age(3600)), timestamping: Some(TimestampingConfig { mode: Some(TimestampingMode::ClientRequire), .. }), .. - } + } if storage_class == "standard" ); Ok(()) @@ -348,7 +348,7 @@ async fn delete_nonexistent_stream_with_ignore(basin: &S2Basin) -> Result<(), Re async fn get_stream_config(basin: &S2Basin) -> Result<(), RequestError> { let stream_name = unique_stream_name(); - let config = StreamConfig::new().with_storage_class(StorageClass::Express); + let config = StreamConfig::new().with_storage_class("express"); basin .create_stream(CreateStreamInput::new(stream_name.clone()).with_config(config)) @@ -356,7 +356,7 @@ async fn get_stream_config(basin: &S2Basin) -> Result<(), RequestError> { let retrieved_config = basin.get_stream_config(stream_name.clone()).await?; - assert_matches!(retrieved_config.storage_class, Some(StorageClass::Express)); + assert_matches!(retrieved_config.storage_class.as_deref(), Some("express")); Ok(()) } @@ -605,7 +605,7 @@ async fn list_all_streams_include_deleted( async fn create_stream_with_full_config(basin: &SharedS2Basin) -> Result<(), RequestError> { let stream_name = unique_stream_name(); let config = StreamConfig::new() - .with_storage_class(StorageClass::Standard) + .with_storage_class("standard") .with_retention_policy(RetentionPolicy::Age(86400)) .with_timestamping( TimestampingConfig::new() @@ -623,7 +623,7 @@ async fn create_stream_with_full_config(basin: &SharedS2Basin) -> Result<(), Req assert_matches!( retrieved, StreamConfig { - storage_class: Some(StorageClass::Standard), + storage_class: Some(ref storage_class), retention_policy: Some(RetentionPolicy::Age(86400)), timestamping: Some(TimestampingConfig { mode: Some(TimestampingMode::ClientRequire), @@ -635,7 +635,7 @@ async fn create_stream_with_full_config(basin: &SharedS2Basin) -> Result<(), Req .. }), .. - } + } if storage_class == "standard" ); basin @@ -649,7 +649,7 @@ async fn create_stream_with_full_config(basin: &SharedS2Basin) -> Result<(), Req #[tokio_shared_rt::test(shared)] async fn create_stream_storage_class_express(basin: &SharedS2Basin) -> Result<(), RequestError> { let stream_name = unique_stream_name(); - let config = StreamConfig::new().with_storage_class(StorageClass::Express); + let config = StreamConfig::new().with_storage_class("express"); let result = basin .create_stream(CreateStreamInput::new(stream_name.clone()).with_config(config)) @@ -664,7 +664,7 @@ async fn create_stream_storage_class_express(basin: &SharedS2Basin) -> Result<() assert_eq!(info.name, stream_name); let retrieved = basin.get_stream_config(stream_name.clone()).await?; - assert_matches!(retrieved.storage_class, Some(StorageClass::Express) | None); + assert_matches!(retrieved.storage_class.as_deref(), Some("express") | None); basin .delete_stream(DeleteStreamInput::new(stream_name)) @@ -853,11 +853,11 @@ async fn reconfigure_stream_storage_class_standard( let config = basin .reconfigure_stream(ReconfigureStreamInput::new( stream_name.clone(), - StreamReconfiguration::new().with_storage_class(StorageClass::Standard), + StreamReconfiguration::new().with_storage_class("standard"), )) .await?; - assert_matches!(config.storage_class, Some(StorageClass::Standard)); + assert_matches!(config.storage_class.as_deref(), Some("standard")); basin .delete_stream(DeleteStreamInput::new(stream_name)) @@ -879,7 +879,7 @@ async fn reconfigure_stream_storage_class_express( let result = basin .reconfigure_stream(ReconfigureStreamInput::new( stream_name.clone(), - StreamReconfiguration::new().with_storage_class(StorageClass::Express), + StreamReconfiguration::new().with_storage_class("express"), )) .await; @@ -894,7 +894,7 @@ async fn reconfigure_stream_storage_class_express( Err(err) => return Err(err), }; - assert_matches!(config.storage_class, Some(StorageClass::Express) | None); + assert_matches!(config.storage_class.as_deref(), Some("express") | None); basin .delete_stream(DeleteStreamInput::new(stream_name)) @@ -1126,7 +1126,7 @@ async fn reconfigure_stream_empty_config_no_change( .create_stream( CreateStreamInput::new(stream_name.clone()).with_config( StreamConfig::new() - .with_storage_class(StorageClass::Standard) + .with_storage_class("standard") .with_retention_policy(RetentionPolicy::Age(3600)), ), ) @@ -1140,7 +1140,7 @@ async fn reconfigure_stream_empty_config_no_change( .await?; let retrieved = basin.get_stream_config(stream_name.clone()).await?; - assert_matches!(retrieved.storage_class, Some(StorageClass::Standard)); + assert_matches!(retrieved.storage_class.as_deref(), Some("standard")); assert_matches!(retrieved.retention_policy, Some(RetentionPolicy::Age(3600))); basin @@ -1229,7 +1229,7 @@ async fn reconfigure_stream_nonexistent_errors(basin: &SharedS2Basin) -> Result< let result = basin .reconfigure_stream(ReconfigureStreamInput::new( unique_stream_name(), - StreamReconfiguration::new().with_storage_class(StorageClass::Standard), + StreamReconfiguration::new().with_storage_class("standard"), )) .await; diff --git a/sdk/tests/stream_ops.rs b/sdk/tests/stream_ops.rs index 54d81180..feb7cad7 100644 --- a/sdk/tests/stream_ops.rs +++ b/sdk/tests/stream_ops.rs @@ -3,7 +3,7 @@ mod common; use std::time::Duration; use assert_matches::assert_matches; -use common::{S2Stream, SharedS2Basin, s2_config, unique_basin_name, unique_stream_name}; +use common::{S2Stream, SharedS2Basin, s2, s2_config, unique_basin_name, unique_stream_name}; use futures_util::{StreamExt, poll}; use rstest::rstest; use s2_sdk::{ @@ -1971,7 +1971,7 @@ async fn create_stream_inherits_basin_default_config() -> Result<(), Box Result<(), Box Result<(), Box> { + let s2 = s2(); + let basin_name = unique_basin_name(); + s2.create_basin( + CreateBasinInput::new(basin_name.clone()).with_config( + BasinConfig::new() + .with_create_stream_on_append(true) + .with_default_stream_config(StreamConfig::new().with_storage_class("standard")), + ), + ) + .await?; + let basin = s2.basin(basin_name.clone()); + + let stream_config = StreamConfig::new() + .with_retention_policy(RetentionPolicy::Age(3600)) + .with_delete_on_empty(DeleteOnEmptyConfig::new().with_min_age(Duration::from_secs(300))); + + let unary_stream = unique_stream_name(); + basin + .stream(unary_stream.clone()) + .append( + AppendInput::new(AppendRecordBatch::try_from_iter([AppendRecord::new( + "hello", + )?])?) + .with_stream_config(stream_config.clone()), + ) + .await?; + let config = basin.get_stream_config(unary_stream).await?; + assert_matches!( + config, + StreamConfig { + storage_class: Some(ref storage_class), + retention_policy: Some(RetentionPolicy::Age(3600)), + delete_on_empty: Some(DeleteOnEmptyConfig { + min_age_secs: 300, + .. + }), + .. + } if storage_class == "standard" + ); + + let session_stream = unique_stream_name(); + let producer = basin + .stream(session_stream.clone()) + .producer(ProducerConfig::new().with_stream_config(stream_config.clone())); + producer.submit(AppendRecord::new("hello")?).await?.await?; + producer.close().await?; + let config = basin.get_stream_config(session_stream).await?; + assert_matches!( + config, + StreamConfig { + retention_policy: Some(RetentionPolicy::Age(3600)), + delete_on_empty: Some(DeleteOnEmptyConfig { + min_age_secs: 300, + .. + }), + .. + } + ); + + let existing_stream = unique_stream_name(); + basin + .create_stream(CreateStreamInput::new(existing_stream.clone())) + .await?; + let before = basin.get_stream_config(existing_stream.clone()).await?; + basin + .stream(existing_stream.clone()) + .append( + AppendInput::new(AppendRecordBatch::try_from_iter([AppendRecord::new( + "hello", + )?])?) + .with_stream_config(stream_config), + ) + .await?; + let after = basin.get_stream_config(existing_stream).await?; + assert_eq!(after, before); + assert_ne!(after.retention_policy, Some(RetentionPolicy::Age(3600))); + + s2.delete_basin(DeleteBasinInput::new(basin_name)).await?; + Ok(()) +} + +#[tokio::test] +async fn stream_config_applies_when_read_creates_stream() -> Result<(), Box> +{ + let s2 = s2(); + let basin_name = unique_basin_name(); + s2.create_basin( + CreateBasinInput::new(basin_name.clone()) + .with_config(BasinConfig::new().with_create_stream_on_read(true)), + ) + .await?; + let basin = s2.basin(basin_name.clone()); + + let stream_config = StreamConfig::new().with_retention_policy(RetentionPolicy::Age(3600)); + + let session_stream = unique_stream_name(); + let _session = basin + .stream(session_stream.clone()) + .read_session( + ReadInput::new().with_stream_config(stream_config), + ReadSessionConfig::default(), + ) + .await?; + let config = basin.get_stream_config(session_stream).await?; + assert_eq!(config.retention_policy, Some(RetentionPolicy::Age(3600))); + + s2.delete_basin(DeleteBasinInput::new(basin_name)).await?; + Ok(()) +} diff --git a/sim/Cargo.lock b/sim/Cargo.lock index a2cf1f39..19ae3e7f 100644 --- a/sim/Cargo.lock +++ b/sim/Cargo.lock @@ -20,9 +20,9 @@ dependencies = [ [[package]] name = "aegis" -version = "0.9.12" +version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e07d39d15384924b35b70d7b8fa1f9a2934101dd3fa4722ede163cc4f9b7b960" +checksum = "58541132f980da31e9aa99f7bdee69bc84bf1e168b9b91ef2dbe8abb7b4ce5dd" dependencies = [ "cc", "softaes", @@ -30,48 +30,36 @@ dependencies = [ [[package]] name = "aes" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138" +checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" dependencies = [ "cipher", "cpubits", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", + "zeroize", ] [[package]] name = "aes-gcm" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" +checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f" dependencies = [ "aead", "aes", "cipher", "ctr", + "ctutils", "ghash", - "subtle", "zeroize", ] -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "getrandom 0.3.4", - "once_cell", - "version_check", - "zerocopy", -] - [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -90,9 +78,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" dependencies = [ "libc", ] @@ -133,7 +121,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -144,7 +132,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -157,7 +145,7 @@ dependencies = [ "libloading", "linkme", "once_cell", - "rand 0.8.6", + "rand 0.8.8", "rand_core 0.6.4", "rustc_version_runtime", "serde", @@ -166,30 +154,24 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arc-swap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" dependencies = [ "rustversion", ] -[[package]] -name = "arrayref" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" - [[package]] name = "arrayvec" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "asn1-rs" @@ -203,7 +185,7 @@ dependencies = [ "nom 7.1.3", "num-traits", "rusticata-macros", - "thiserror 2.0.18", + "thiserror 2.0.20", "time", ] @@ -215,7 +197,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] @@ -227,7 +209,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -244,9 +226,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.42" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79b3f8a79cccc2898f31920fc69f304859b3bd567490f75ebf51ae1c792a9ac" +checksum = "4f10dafd0c8d2e51ae9a748805777613ed0bbe17bf586b76c8311f45c020a32f" dependencies = [ "compression-codecs", "compression-core", @@ -273,20 +255,26 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] +[[package]] +name = "asyncband" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2d85fd3d291fabcc40c7232c92c280ec1754fd7b5d7ea769f143222143e179a" + [[package]] name = "atoi" version = "3.1.0" @@ -319,9 +307,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.8.15" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11493b0bad143270fb8ad284a096dd529ba91924c5409adeac856cc1bf047dbc" +checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" dependencies = [ "aws-credential-types", "aws-runtime", @@ -333,13 +321,14 @@ dependencies = [ "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", "fastrand", "hex", - "http 1.4.2", - "sha1 0.10.6", + "http 1.5.0", + "sha1 0.10.7", "time", "tokio", "tracing", @@ -349,9 +338,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.14" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7" +checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -361,9 +350,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.17.0" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -372,21 +361,22 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.41.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] name = "aws-runtime" -version = "1.7.2" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fc0651c57e384202e47153c1260b84a9936e19803d747615edf199dc3b98d17" +checksum = "ef47857a1d4488b528f4a5d5715fa7c3300820897824152234d3fa22b1426657" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -399,8 +389,8 @@ dependencies = [ "bytes", "bytes-utils", "fastrand", - "http 1.4.2", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "percent-encoding", "pin-project-lite", "tracing", @@ -409,10 +399,11 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.97.0" +version = "1.109.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aadc669e184501caaa6beafb28c6267fc1baef0810fb58f9b205485ca3f2567" +checksum = "c3cfe74df5d9ad2fedd691973ad3521ebf4f27a3c68c792556686aedb5519bab" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", @@ -421,22 +412,24 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", "fastrand", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-ssooidc" -version = "1.99.0" +version = "1.111.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1342a7db8f358d3de0aed2007a0b54e875458e39848d54cc1d46700b2bfcb0a8" +checksum = "81b0ec31ed6191bd11350aae4b2004198f2db21350cb0a20c57e0a92e55dd161" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", @@ -445,22 +438,24 @@ dependencies = [ "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-types", "bytes", "fastrand", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-sts" -version = "1.102.0" +version = "1.114.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc35b7a14cabdad13795fbbbd26d5ddec0882c01492ceedf2af575aad5f37dd" +checksum = "ef45745026107ec30c4ef86bd8ae4b002e7e5f6a86e4225240bdf6b06a0b944a" dependencies = [ + "arc-swap", "aws-credential-types", "aws-runtime", "aws-smithy-async", @@ -470,21 +465,22 @@ dependencies = [ "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", "aws-smithy-xml", "aws-types", "fastrand", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "regex-lite", "tracing", ] [[package]] name = "aws-sigv4" -version = "1.4.2" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4" +checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -493,20 +489,20 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac 0.12.1", + "hmac", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "percent-encoding", - "sha2 0.10.9", + "sha2", "time", "tracing", ] [[package]] name = "aws-smithy-async" -version = "1.2.14" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc" +checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" dependencies = [ "futures-util", "pin-project-lite", @@ -515,9 +511,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.63.6" +version = "0.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231" +checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -525,8 +521,8 @@ dependencies = [ "bytes-utils", "futures-core", "futures-util", - "http 1.4.2", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "http-body-util", "percent-encoding", "pin-project-lite", @@ -536,15 +532,15 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.1.13" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3ef8931ad1c98aa6a55b4256f847f3116090819844e0dd41ea682cac5dd2d3" +checksum = "ebfd138fac0337cee7516c352757ea73b9f2266e57d0bcb5bc70e9547e45aef1" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", "h2", - "http 1.4.2", + "http 1.5.0", "hyper", "hyper-rustls", "hyper-util", @@ -560,9 +556,9 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.62.7" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "701a947f4797e52a911e114a898667c746c39feea467bbd1abd7b3721f702ffa" +checksum = "3dc65a121adb4b33729919fcfa14fa36fb33c1555a8f06bb0e2188dbfdc1d9ef" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-schema", @@ -571,28 +567,31 @@ dependencies = [ [[package]] name = "aws-smithy-observability" -version = "0.2.6" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c" +checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" dependencies = [ "aws-smithy-runtime-api", ] [[package]] name = "aws-smithy-query" -version = "0.60.15" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd" +checksum = "512346c7212ab7436df2d77a16d976a468ae44a418835511d2a69269810aaf62" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", "aws-smithy-types", + "aws-smithy-xml", "urlencoding", ] [[package]] name = "aws-smithy-runtime" -version = "1.11.3" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e6f5caf6fea86f8c2206541ab5857cfcda9013426cdbe8fa0098b9e2d32182" +checksum = "b82e438d30e02a825d363bd639a9efaed68a8089d86101054b0081e7e0d3e606" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -604,9 +603,9 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "http-body 0.4.6", - "http-body 1.0.1", + "http-body 1.1.0", "http-body-util", "pin-project-lite", "pin-utils", @@ -616,16 +615,16 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.12.3" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9db177daa6ba8afb9ee1aefcf548c907abcf52065e394ee11a92780057fe0e8c" +checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "pin-project-lite", "tokio", "tracing", @@ -634,39 +633,39 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api-macros" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d7396fd9500589e62e460e987ecb671bad374934e55ec3b5f498cc7a8a8a7b7" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "aws-smithy-schema" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7442cb268338f0eb8278140a107c046756aa01093d8ef5e99628d34ae09c94f5" +checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", - "http 1.4.2", + "http 1.5.0", ] [[package]] name = "aws-smithy-types" -version = "1.5.0" +version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b42fcf341259d85ca10fac9a2f6448a8ec691c6955a18e45bc3b71a85fab85" +checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9" dependencies = [ "base64-simd", "bytes", "bytes-utils", "http 0.2.12", - "http 1.4.2", + "http 1.5.0", "http-body 0.4.6", - "http-body 1.0.1", + "http-body 1.1.0", "http-body-util", "itoa", "num-integer", @@ -679,18 +678,21 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.15" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" +checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8" dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", "xmlparser", ] [[package]] name = "aws-types" -version = "1.3.16" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16bf10b03a3c01e6b3b7d47cd964e873ffe9e7d4e80fad16bd4c077cb068531" +checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -712,8 +714,8 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http 1.4.2", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "http-body-util", "hyper", "hyper-util", @@ -743,8 +745,8 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http 1.4.2", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "http-body-util", "mime", "pin-project-lite", @@ -762,7 +764,7 @@ checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -775,8 +777,8 @@ dependencies = [ "bytes", "either", "fs-err", - "http 1.4.2", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "hyper", "hyper-util", "pin-project-lite", @@ -804,6 +806,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -840,22 +848,21 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "blake3" -version = "1.8.5" +version = "1.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" dependencies = [ - "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", ] [[package]] @@ -884,15 +891,15 @@ checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" [[package]] name = "bytemuck" -version = "1.25.0" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" [[package]] name = "bytes" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" dependencies = [ "serde", ] @@ -909,9 +916,9 @@ dependencies = [ [[package]] name = "bytesize" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "351a3e803ee3c6eaeee6b00076b767514b37c32a73d326c3ec7abddb7d6c3493" +checksum = "7354288c522e7e980fafd2075d63d1285794c3a6a16cdd492f189ea406e5f18b" [[package]] name = "bytestring" @@ -933,9 +940,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.64" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" +checksum = "005ec2760ca554fae18df7a11195552ec576cd665632a881bc011d5bb2fd4d80" dependencies = [ "find-msvc-tools", "jobserver", @@ -951,18 +958,18 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" [[package]] name = "chacha20" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -993,9 +1000,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", "clap_derive", @@ -1003,9 +1010,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstream", "anstyle", @@ -1015,14 +1022,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] @@ -1046,12 +1053,6 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" -[[package]] -name = "cmsketch" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7ee2cfacbd29706479902b06d75ad8f1362900836aa32799eabc7e004bfd854" - [[package]] name = "colorchoice" version = "1.0.5" @@ -1060,9 +1061,9 @@ checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" [[package]] name = "combine" -version = "4.6.7" +version = "4.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" dependencies = [ "bytes", "memchr", @@ -1084,22 +1085,22 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.38" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +checksum = "58a6d0db8759036a783bc7c3f7a07f8cef3bf9470eb1db3bc86e8bcd1c5d0fe8" dependencies = [ "compression-core", "flate2", "memchr", - "zstd", - "zstd-safe", + "zstd 0.14.0", + "zstd-safe 8.0.0", ] [[package]] name = "compression-core" -version = "0.4.32" +version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" +checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" [[package]] name = "concurrent-queue" @@ -1166,9 +1167,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] @@ -1185,18 +1186,18 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", ] @@ -1213,9 +1214,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crypto-common" @@ -1276,7 +1277,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1287,7 +1288,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1306,9 +1307,15 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "datasketches" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" [[package]] name = "der-parser" @@ -1341,7 +1348,6 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "crypto-common 0.1.7", - "subtle", ] [[package]] @@ -1358,13 +1364,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] @@ -1387,7 +1393,7 @@ checksum = "f88959de2d447fd3eddcf1909d1f19fe084e27a056a6904203dc5d8b9e771c1e" dependencies = [ "rust_decimal", "serde", - "thiserror 2.0.18", + "thiserror 2.0.20", "time", "winnow 0.6.26", ] @@ -1400,15 +1406,15 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" -version = "1.16.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" [[package]] name = "enumset" -version = "1.1.13" +version = "1.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "839c4174b41e75c8f7306110b2c51996a293b8d1d850edd529011841d9fede7d" +checksum = "ccc5801fd11762e24d1e420d01d2ac518f2a2ca4329d4fbb6639f2412b6204e0" dependencies = [ "enumset_derive", ] @@ -1422,7 +1428,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1438,16 +1444,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "event-listener" -version = "5.4.1" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "concurrent-queue", "parking", "pin-project-lite", ] @@ -1464,10 +1469,11 @@ dependencies = [ [[package]] name = "eyre" -version = "0.6.12" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +checksum = "c08309dbcc659c5549a24ddb9b27027640641b282ef5768267c7e675558986a3" dependencies = [ + "autocfg", "indenter", "once_cell", ] @@ -1480,7 +1486,7 @@ checksum = "c29b33a0187823f1fa88b36980227dc96c7504ede2288e7d2a77d9d6d88b260c" dependencies = [ "log", "once_cell", - "rand 0.9.4", + "rand 0.9.5", "tokio", ] @@ -1496,21 +1502,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" - -[[package]] -name = "faststr" -version = "0.2.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca7d44d22004409a61c393afb3369c8f7bb74abcae49fe249ee01dcc3002113" -dependencies = [ - "bytes", - "rkyv", - "serde", - "simdutf8", -] +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "figment" @@ -1530,9 +1524,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" [[package]] name = "flatbuffers" @@ -1546,12 +1540,13 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -1577,18 +1572,18 @@ dependencies = [ [[package]] name = "foyer" -version = "0.22.3" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0abc0b87814989efa711f9becd9f26969820e2d3905db27d10969c4bd45890" +checksum = "6a59f42276891c0a4ce683fcda1aa1b4fd1065d68116c264e4bf49b9d318a0ed" dependencies = [ "anyhow", + "asyncband", "equivalent", "foyer-common", "foyer-memory", "foyer-storage", "foyer-tokio", "futures-util", - "mea", "mixtrics", "pin-project", "serde", @@ -1597,9 +1592,9 @@ dependencies = [ [[package]] name = "foyer-common" -version = "0.22.3" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3db80d5dece93adb7ad709c84578794724a9cba342a7e566c3551c7ec626789" +checksum = "643b47d510032a01e5af70dca288b0c5ec531646c1a8392e793fb5b0c13bef30" dependencies = [ "anyhow", "bincode", @@ -1624,21 +1619,21 @@ dependencies = [ [[package]] name = "foyer-memory" -version = "0.22.3" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db907f40a527ca2aa2f40a5f68b32ea58aa70f050cd233518e9ffd402cfba6ce" +checksum = "ae51f5089f3d9025ae77f17ea66d2489ac0f82fbb1d91462807bea174d486d82" dependencies = [ "anyhow", + "asyncband", "bitflags", - "cmsketch", + "datasketches", "equivalent", "foyer-common", "foyer-intrusive-collections", "foyer-tokio", "futures-util", - "hashbrown 0.16.1", - "itertools 0.14.0", - "mea", + "hashbrown 0.17.1", + "itertools 0.15.0", "mixtrics", "parking_lot", "paste", @@ -1649,12 +1644,13 @@ dependencies = [ [[package]] name = "foyer-storage" -version = "0.22.3" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1983f1db3d0710e9c9d5fc116d9202dccd41a2d1e032572224f1aff5520aa958" +checksum = "118192f532ba013f0cdc607efc22324b9ed855baed185608af0daa986e835c6b" dependencies = [ "allocator-api2", "anyhow", + "asyncband", "bytes", "core_affinity", "equivalent", @@ -1665,35 +1661,34 @@ dependencies = [ "fs4", "futures-core", "futures-util", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "io-uring", - "itertools 0.14.0", + "itertools 0.15.0", "libc", "lz4", - "mea", "parking_lot", "pin-project", - "rand 0.9.4", + "rand 0.10.2", "serde", "tracing", "twox-hash", - "zstd", + "zstd 0.13.3", ] [[package]] name = "foyer-tokio" -version = "0.22.3" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6577b05a7ffad0db555aedf00bfe52af818220fc4c1c3a7a12520896fc38627" +checksum = "6741d1133dfaab64d3f8a9290bbfed3685084add28f8b6ee7a6264aa4dc26918" dependencies = [ "tokio", ] [[package]] name = "fs-err" -version = "3.3.0" +version = "3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fde052dbfc920003cfd2c8e2c6e6d4cc7c1091538c3a24226cec0665ab08c0" +checksum = "b91aa448ca50d7e79433bdf3ee8d99215430d2ec02ade5aefab2a073a1822e8a" dependencies = [ "autocfg", "tokio", @@ -1717,9 +1712,9 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -1732,9 +1727,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -1742,15 +1737,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -1759,38 +1754,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -1823,7 +1818,7 @@ dependencies = [ "libc", "wasi", "wasm-bindgen", - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -1833,11 +1828,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -1847,9 +1840,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -1859,6 +1854,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" dependencies = [ "polyval", + "zeroize", ] [[package]] @@ -1875,16 +1871,16 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.16" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.4.2", + "http 1.5.0", "indexmap", "slab", "tokio", @@ -1904,17 +1900,6 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -[[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] - [[package]] name = "hashbrown" version = "0.17.1" @@ -1940,9 +1925,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" [[package]] name = "hex" @@ -1960,15 +1945,6 @@ dependencies = [ "vsimd", ] -[[package]] -name = "hmac" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" -dependencies = [ - "digest 0.10.7", -] - [[package]] name = "hmac" version = "0.13.0" @@ -1991,9 +1967,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -2012,24 +1988,24 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", - "http 1.4.2", + "http 1.5.0", ] [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", - "http 1.4.2", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "pin-project-lite", ] @@ -2053,26 +2029,26 @@ checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hybrid-array" -version = "0.4.12" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" dependencies = [ "typenum", ] [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", "h2", - "http 1.4.2", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "httparse", "httpdate", "itoa", @@ -2088,7 +2064,7 @@ version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http 1.4.2", + "http 1.5.0", "hyper", "hyper-util", "rustls", @@ -2104,12 +2080,12 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", - "http 1.4.2", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "hyper", "ipnet", "libc", @@ -2147,9 +2123,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -2161,9 +2137,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -2174,9 +2150,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -2188,16 +2164,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -2208,15 +2185,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", @@ -2262,9 +2239,9 @@ checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -2287,9 +2264,9 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.12" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" +checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" dependencies = [ "bitflags", "cfg-if", @@ -2298,9 +2275,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.12.0" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "is_terminal_polyfill" @@ -2344,7 +2321,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.18", + "thiserror 2.0.20", "walkdir", "windows-link 0.2.1", ] @@ -2359,7 +2336,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn", + "syn 2.0.119", ] [[package]] @@ -2378,24 +2355,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "jobserver" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom 0.3.4", + "getrandom 0.4.3", "libc", ] [[package]] name = "js-sys" -version = "0.3.102" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" dependencies = [ "cfg-if", "futures-util", @@ -2410,9 +2387,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libloading" @@ -2432,22 +2409,22 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "linkme" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83272d46373fb8decca684579ac3e7c8f3d71d4cc3aa693df8759e260ae41cf" +checksum = "3045e122bd98aef8ec3ad58ce84f0791f64e70163d1a02710af4aa11a4d54cc5" dependencies = [ "linkme-impl", ] [[package]] name = "linkme-impl" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d59e20403c7d08fe62b4376edfe5c7fb2ef1e6b1465379686d0f21c8df444b" +checksum = "77060ebe535362c3da75682cd17b0431017b6e7c5661e714fc69a7ad017d1301" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] @@ -2464,9 +2441,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "lock_api" @@ -2479,15 +2456,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.32" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lru" -version = "0.18.2" +version = "0.18.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" +checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" dependencies = [ "hashbrown 0.17.1", ] @@ -2533,7 +2510,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06138aebc1442e86b201f492616a51690a758d7b75253162f9e9208382ca2bb7" dependencies = [ "libc", - "rand 0.10.1", + "rand 0.10.2", "tokio", "turmoil", ] @@ -2563,20 +2540,11 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "mea" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2640d335e7273dacdcf51044026139b2e269c3bb0dfc3f8cb3496b85e3f6a42c" -dependencies = [ - "slab", -] - [[package]] name = "memchr" -version = "2.8.2" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memoffset" @@ -2601,9 +2569,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.9" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -2611,9 +2579,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.1" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", "wasi", @@ -2630,26 +2598,6 @@ dependencies = [ "parking_lot", ] -[[package]] -name = "munge" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e17401f259eba956ca16491461b6e8f72913a0a114e39736ce404410f915a0c" -dependencies = [ - "munge_macro", -] - -[[package]] -name = "munge_macro" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4568f25ccbd45ab5d5603dc34318c1ec56b117531781260002151b8530a9f931" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "nix" version = "0.31.3" @@ -2696,14 +2644,14 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", @@ -2717,9 +2665,9 @@ checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" dependencies = [ "num-traits", ] @@ -2771,13 +2719,13 @@ dependencies = [ [[package]] name = "object_store" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "765784b4390c6bcf80316e5a22f4e3661b639c9d8c83246856643c27d8ce9dbe" +checksum = "d354792e39fa5f0009e47623cf8b15b099bf9a652fa55c6f817fe28ac84fea50" dependencies = [ "async-trait", "aws-lc-rs", - "base64", + "base64 0.22.1", "bytes", "chrono", "crc-fast", @@ -2785,7 +2733,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body-util", "humantime", "hyper", @@ -2794,14 +2742,14 @@ dependencies = [ "nix", "parking_lot", "percent-encoding", - "quick-xml 0.40.1", - "rand 0.10.1", + "quick-xml", + "rand 0.10.2", "reqwest", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", "url", @@ -2859,7 +2807,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2923,16 +2871,16 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "pem" -version = "3.0.6" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" dependencies = [ - "base64", + "base64 0.23.1", "serde_core", ] @@ -2959,7 +2907,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2976,26 +2924,27 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "polyval" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfc63250416fea14f5749b90725916a6c903f599d51cb635aa7a52bfd03eede" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" dependencies = [ "cpubits", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "universal-hash", + "zeroize", ] [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -3017,9 +2966,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] @@ -3032,7 +2981,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "version_check", "yansi", ] @@ -3073,7 +3022,7 @@ dependencies = [ "parking_lot", "procfs", "protobuf", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -3096,7 +3045,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3119,36 +3068,6 @@ dependencies = [ "thiserror 1.0.69", ] -[[package]] -name = "ptr_meta" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9a0cf95a1196af61d4f1cbdab967179516d9a4a4312af1f31948f8f6224a79" -dependencies = [ - "ptr_meta_derive", -] - -[[package]] -name = "ptr_meta_derive" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7347867d0a7e1208d93b46767be83e2b8f978c3dad35f775ac8d8847551d6fe1" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "quick-xml" -version = "0.40.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quick-xml" version = "0.41.0" @@ -3161,9 +3080,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.9" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ "bytes", "cfg_aliases", @@ -3173,7 +3092,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.18", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -3181,21 +3100,22 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.4", + "rand 0.10.2", + "rand_pcg", "ring", "rustc-hash", "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.18", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -3203,23 +3123,23 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" dependencies = [ "cfg_aliases", "libc", "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] @@ -3236,20 +3156,11 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rancor" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a063ea72381527c2a0561da9c80000ef822bdd7c3241b1cc1b12100e3df081ee" -dependencies = [ - "ptr_meta", -] - [[package]] name = "rand" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -3258,9 +3169,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -3268,9 +3179,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -3328,7 +3239,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.4", + "rand 0.9.5", +] + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", ] [[package]] @@ -3342,9 +3262,9 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.14.8" +version = "0.14.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055" +checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" dependencies = [ "aws-lc-rs", "pem", @@ -3365,29 +3285,41 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.25" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.25" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", ] [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -3406,25 +3338,19 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" -[[package]] -name = "rend" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cadadef317c2f20755a64d7fdc48f9e7178ee6b0e1f7fce33fa60f1d68a276e6" - [[package]] name = "reqwest" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64", + "base64 0.23.1", "bytes", "futures-core", "futures-util", "h2", - "http 1.4.2", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "http-body-util", "hyper", "hyper-rustls", @@ -3465,40 +3391,11 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rkyv" -version = "0.8.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73389e0c99e664f919275ab5b5b0471391fe9a8de61e1dff9b1eaf56a90f16e3" -dependencies = [ - "bytes", - "hashbrown 0.17.1", - "indexmap", - "munge", - "ptr_meta", - "rancor", - "rend", - "rkyv_derive", - "tinyvec", - "uuid", -] - -[[package]] -name = "rkyv_derive" -version = "0.8.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2ed0b54125315fb36bd021e82d314d1c126548f871634b483f46b31d13cac6" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "rust_decimal" -version = "1.42.1" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be2a24f50780bc85f09cc6ac299bdf1424302742d77221106859c9d8b102126a" +checksum = "7653272e75dcac41dc199fbea6f5797633994fafd339943c06c9af16bf29cd3a" dependencies = [ "arrayvec", "num-traits", @@ -3506,9 +3403,9 @@ dependencies = [ [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -3561,14 +3458,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "log", @@ -3593,9 +3490,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "web-time", "zeroize", @@ -3619,7 +3516,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3630,9 +3527,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -3642,9 +3539,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "ryu" @@ -3654,7 +3551,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "s2-api" -version = "0.31.1" +version = "0.31.5" dependencies = [ "axum", "base64ct", @@ -3663,24 +3560,23 @@ dependencies = [ "flate2", "futures-core", "futures-util", - "http 1.4.2", + "http 1.5.0", "itertools 0.15.0", "mime", "prost", "s2-common", "serde", "serde_json", - "sonic-rs", "strum", - "thiserror 2.0.18", + "thiserror 2.0.20", "time", "tokio-util", - "zstd", + "zstd 0.14.0", ] [[package]] name = "s2-common" -version = "0.41.1" +version = "0.41.3" dependencies = [ "axum", "base64ct", @@ -3688,18 +3584,18 @@ dependencies = [ "clap", "compact_str", "enumset", - "http 1.4.2", - "rand 0.10.1", + "http 1.5.0", + "rand 0.10.2", "secrecy", "serde", "strum", - "thiserror 2.0.18", + "thiserror 2.0.20", "time", ] [[package]] name = "s2-lite" -version = "0.42.7" +version = "0.42.13" dependencies = [ "async-stream", "async-trait", @@ -3713,13 +3609,13 @@ dependencies = [ "dashmap", "eyre", "futures", - "http 1.4.2", + "http 1.5.0", "indexmap", "itertools 0.15.0", "parking_lot", "prometheus", "prost", - "rand 0.10.1", + "rand 0.10.2", "rcgen", "rustls", "s2-api", @@ -3730,12 +3626,12 @@ dependencies = [ "serde_json", "slatedb", "strum", - "thiserror 2.0.18", + "thiserror 2.0.20", "tikv-jemallocator", "time", "tokio", "tokio-util", - "tower-http 0.7.0", + "tower-http 0.7.1", "tracing", "tracing-subscriber", ] @@ -3744,6 +3640,7 @@ dependencies = [ name = "s2-resource-spec" version = "0.2.2" dependencies = [ + "compact_str", "humantime", "s2-common", "schemars", @@ -3753,24 +3650,25 @@ dependencies = [ [[package]] name = "s2-sdk" -version = "0.34.3" +version = "0.34.9" dependencies = [ "async-compression", "async-stream", "async-trait", "bytes", + "compact_str", "futures-core", "futures-util", "h2", - "http 1.4.2", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "http-body-util", "hyper", "hyper-rustls", "hyper-util", "pin-project-lite", "prost", - "rand 0.10.1", + "rand 0.10.2", "rustls", "s2-api", "s2-common", @@ -3778,7 +3676,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "thiserror 2.0.18", + "thiserror 2.0.20", "time", "tokio", "tokio-muxt", @@ -3803,12 +3701,12 @@ dependencies = [ "fastrand", "futures", "getrandom 0.2.15", - "http 1.4.2", + "http 1.5.0", "http-body-util", "hyper", "hyper-util", "mad-turmoil", - "rand 0.10.1", + "rand 0.10.2", "s2-lite", "s2-sdk", "s2-verification", @@ -3824,17 +3722,17 @@ dependencies = [ [[package]] name = "s2-storage" -version = "0.2.4" +version = "0.2.5" dependencies = [ "aegis", "aes-gcm", "blake3", "bytes", - "rand 0.10.1", + "rand 0.10.2", "s2-common", "secrecy", "serde", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -3846,7 +3744,7 @@ dependencies = [ "clap", "eyre", "futures", - "rand 0.8.6", + "rand 0.8.8", "s2-sdk", "serde", "serde_json", @@ -3859,9 +3757,9 @@ dependencies = [ [[package]] name = "s3s" -version = "0.14.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe1bd31748cb69848c2cf4028cecdadf5f8299ef3b02a83e21b20e7bda3aba9" +checksum = "652451a66fefda01a13dc7df24aa2af9e70c7058f98bc08fe2f7c552eaa9f1e3" dependencies = [ "arc-swap", "arrayvec", @@ -3875,9 +3773,9 @@ dependencies = [ "crc-fast", "futures", "hex-simd", - "hmac 0.13.0", - "http 1.4.2", - "http-body 1.0.1", + "hmac", + "http 1.5.0", + "http-body 1.1.0", "http-body-util", "httparse", "hyper", @@ -3888,17 +3786,18 @@ dependencies = [ "nom 8.0.0", "numeric_cast", "pin-project-lite", - "quick-xml 0.41.0", + "quick-xml", + "regex", "serde", "serde_json", "serde_urlencoded", "sha1 0.11.0", - "sha2 0.11.0", + "sha2", "smallvec", "std-next", "subtle", "sync_wrapper", - "thiserror 2.0.18", + "thiserror 2.0.20", "time", "tokio", "tower", @@ -3906,6 +3805,7 @@ dependencies = [ "transform-stream", "url", "urlencoding", + "xxhash-rust", "zeroize", ] @@ -3929,9 +3829,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" dependencies = [ "dyn-clone", "ref-cast", @@ -3942,14 +3842,14 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d115b50f4aaeea07e79c1912f645c7513d81715d0420f8bc77a18c6260b307f" +checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 3.0.5", ] [[package]] @@ -4004,9 +3904,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -4014,40 +3914,40 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] name = "serde_derive_internals" -version = "0.29.1" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -4103,9 +4003,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", "cpufeatures 0.2.17", @@ -4119,21 +4019,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "digest 0.11.3", ] -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest 0.10.7", -] - [[package]] name = "sha2" version = "0.11.0" @@ -4141,7 +4030,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "digest 0.11.3", ] @@ -4172,15 +4061,15 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" [[package]] name = "simd_cesu8" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" dependencies = [ "rustc_version", "simdutf8", @@ -4206,9 +4095,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "slatedb" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35ca56b01922b15aa69fe3abb62cadc985d86032c9647e4606e211c4da751a76" +checksum = "fb40332f41e231926df3a0ef742c05316b43368ba4b520433d2a1eba1ab00f0f" dependencies = [ "async-channel", "async-trait", @@ -4232,7 +4121,7 @@ dependencies = [ "object_store", "ouroboros", "parking_lot", - "rand 0.9.4", + "rand 0.9.5", "serde", "serde_json", "siphasher", @@ -4248,19 +4137,19 @@ dependencies = [ "url", "uuid", "walkdir", - "zstd", + "zstd 0.13.3", ] [[package]] name = "slatedb-common" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa8de522ff46a0f9b5a66f45e650d125421d4d91990933591092f9d010c40d1" +checksum = "6b77f238f348e95e1653a5235f880ac703011b82582bd556f2aba405abf8180e" dependencies = [ "chrono", "log", "object_store", - "rand 0.9.4", + "rand 0.9.5", "rand_xoshiro", "serde", "thread_local", @@ -4269,9 +4158,9 @@ dependencies = [ [[package]] name = "slatedb-txn-obj" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85fd9c0c86dd4954524fa8238d0548bd80f4a9a66983cbde3728402d339993e" +checksum = "1ad63b6c250219e26d71f497820f970274c301f896366daa56915f2a05df0e0a" dependencies = [ "async-trait", "bytes", @@ -4292,18 +4181,18 @@ checksum = "88414a5ca1f85d82cc34471e975f0f74f6aa54c40f062efa42c0080e7f763f81" [[package]] name = "smallvec" -version = "1.15.2" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" [[package]] name = "socket2" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4312,50 +4201,11 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45e14297decde697ddf377c25752aead0927d5cfc89c2684d2af96901a4ceeea" -[[package]] -name = "sonic-number" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3775c3390edf958191f1ab1e8c5c188907feebd0f3ce1604cb621f72961dbf32" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "sonic-rs" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d971cc77a245ccf1756dbd1a87c3e7f709c0191464096510d43eec056d0f2c4f" -dependencies = [ - "ahash", - "bumpalo", - "bytes", - "cfg-if", - "faststr", - "itoa", - "ref-cast", - "serde", - "simdutf8", - "sonic-number", - "sonic-simd", - "thiserror 2.0.18", - "zmij", -] - -[[package]] -name = "sonic-simd" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f99e664ecd2d85a68c87e3c7a3cfe691f647ea9e835de984aba4d54a41f817d4" -dependencies = [ - "cfg-if", -] - [[package]] name = "spin" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" [[package]] name = "stable_deref_trait" @@ -4376,7 +4226,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04082e93ed1a06debd9148c928234b46d2cf260bc65f44e1d1d3fa594c5beebc" dependencies = [ "simdutf8", - "thiserror 2.0.18", + "thiserror 2.0.20", ] [[package]] @@ -4403,7 +4253,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -4414,9 +4264,20 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.118" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" dependencies = [ "proc-macro2", "quote", @@ -4440,7 +4301,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -4468,11 +4329,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl 2.0.20", ] [[package]] @@ -4483,25 +4344,25 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] name = "thread_local" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" dependencies = [ "cfg-if", ] @@ -4528,9 +4389,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.49" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", "num-conv", @@ -4548,9 +4409,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.29" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -4558,9 +4419,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -4568,9 +4429,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.11.0" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" dependencies = [ "tinyvec_macros", ] @@ -4600,13 +4461,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] @@ -4621,9 +4482,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.4" +version = "0.26.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" dependencies = [ "rustls", "tokio", @@ -4631,9 +4492,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" dependencies = [ "futures-core", "pin-project-lite", @@ -4642,15 +4503,16 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", "futures-util", "hashbrown 0.15.5", + "libc", "pin-project-lite", "tokio", ] @@ -4721,8 +4583,8 @@ dependencies = [ "bitflags", "bytes", "futures-util", - "http 1.4.2", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "pin-project-lite", "tower", "tower-layer", @@ -4732,16 +4594,16 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233" +checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c" dependencies = [ "async-compression", "bitflags", "bytes", "futures-core", - "http 1.4.2", - "http-body 1.0.1", + "http 1.5.0", + "http-body 1.1.0", "http-body-util", "percent-encoding", "pin-project-lite", @@ -4784,7 +4646,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -4862,7 +4724,7 @@ checksum = "0061c28f1469e94771bb8aa626ce6b29265c2fb5034115046a170b0cba2e33d2" dependencies = [ "bytes", "indexmap", - "rand 0.9.4", + "rand 0.9.5", "rand_distr", "scoped-tls", "tokio", @@ -4871,12 +4733,9 @@ dependencies = [ [[package]] name = "twox-hash" -version = "2.1.2" +version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" -dependencies = [ - "rand 0.9.4", -] +checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" [[package]] name = "typenum" @@ -4890,7 +4749,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" dependencies = [ - "rand 0.9.4", + "rand 0.9.5", "serde", "web-time", ] @@ -4970,13 +4829,13 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.3" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ "getrandom 0.4.3", "js-sys", - "rand 0.10.1", + "rand 0.10.2", "serde_core", "wasm-bindgen", ] @@ -5035,9 +4894,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.125" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" dependencies = [ "cfg-if", "once_cell", @@ -5048,9 +4907,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.75" +version = "0.4.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" dependencies = [ "js-sys", "wasm-bindgen", @@ -5058,9 +4917,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.125" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5068,22 +4927,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.125" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 3.0.5", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.125" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" dependencies = [ "unicode-ident", ] @@ -5103,9 +4962,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.102" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" dependencies = [ "js-sys", "wasm-bindgen", @@ -5123,9 +4982,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" dependencies = [ "rustls-pki-types", ] @@ -5152,7 +5011,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5228,7 +5087,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -5239,7 +5098,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -5306,7 +5165,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -5315,16 +5174,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", + "windows-targets", ] [[package]] @@ -5342,31 +5192,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link 0.2.1", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -5384,96 +5217,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "winnow" version = "0.6.26" @@ -5500,9 +5285,9 @@ checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "x509-parser" @@ -5518,7 +5303,7 @@ dependencies = [ "nom 7.1.3", "oid-registry", "rusticata-macros", - "thiserror 2.0.18", + "thiserror 2.0.20", "time", ] @@ -5530,9 +5315,9 @@ checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "xxhash-rust" -version = "0.8.15" +version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" [[package]] name = "yansi" @@ -5569,28 +5354,28 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.52" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +checksum = "d35102a9f36d089ccae9e4c6802bc118be4487b80aaffc0ab4e0cf5ce92d2873" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.52" +version = "0.8.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +checksum = "146c01f5ab44258da43cf276c74a2763db2ff3969c9c652c3f2de07041d0b2bc" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -5610,7 +5395,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", "synstructure", ] @@ -5622,9 +5407,9 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -5633,9 +5418,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ "yoke", "zerofrom", @@ -5644,20 +5429,26 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.5", ] [[package]] -name = "zmij" -version = "1.0.21" +name = "zlib-rs" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] name = "zstd" @@ -5665,23 +5456,41 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe", + "zstd-safe 7.3.0", +] + +[[package]] +name = "zstd" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e" +dependencies = [ + "zstd-safe 8.0.0", ] [[package]] name = "zstd-safe" -version = "7.2.4" +version = "7.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-safe" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.0+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" dependencies = [ "cc", "pkg-config", diff --git a/sim/Cargo.toml b/sim/Cargo.toml index 2b537aad..1293bd1e 100644 --- a/sim/Cargo.toml +++ b/sim/Cargo.toml @@ -19,17 +19,17 @@ path = "src/main.rs" async-trait = "0.1" axum = "0.8" blake3 = "1.8" -bytes = "1.11" -bytesize = "2.3" +bytes = "1.12" +bytesize = "2.7" clap = { version = "4.6", features = ["derive", "env"] } eyre = "0.6" fastrand = "2" futures = "0.3" # Activates determinism on Linux in the patched getrandom (see [patch.crates-io]). getrandom = { version = "0.2", features = ["deterministic-simulation"] } -http = "1.4" +http = "1.5" http-body-util = "0.1" -hyper = { version = "1.9", features = ["client", "server", "http1", "http2"] } +hyper = { version = "1.11", features = ["client", "server", "http1", "http2"] } hyper-util = { version = "0.1", features = [ "client-legacy", "http1", @@ -46,10 +46,10 @@ s2-sdk = { path = "../sdk", features = ["_hidden"] } # The Go checker built in CI must be pinned to the same rev (see the # S2_VERIFICATION_REV env in .github/workflows/ci.yml). s2-verification = { git = "https://github.com/s2-streamstore/s2-verification", rev = "b4af8c8ef4965d9b335101c422eadb33f3169004" } -s3s = "0.14.1" +s3s = "0.15.0" serde_json = "1.0" -slatedb = { version = "0.15.0", features = ["lz4", "zstd"] } -tokio = { version = "1.52", features = ["macros", "rt", "sync", "time", "io-util"] } +slatedb = { version = "0.16.0", features = ["lz4", "zstd"] } +tokio = { version = "1.53", features = ["macros", "rt", "sync", "time", "io-util"] } tower = "0.5" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/storage/CHANGELOG.md b/storage/CHANGELOG.md index 3719b3b7..aeb91386 100644 --- a/storage/CHANGELOG.md +++ b/storage/CHANGELOG.md @@ -2,6 +2,22 @@ All notable changes to this project will be documented in this file. +## [0.2.6] - 2026-09-25 + +### Miscellaneous Tasks + +- Updated the following local packages: s2-common + + + +## [0.2.5] - 2026-09-22 + +### Miscellaneous Tasks + +- Update Cargo.toml dependencies + + + ## [0.2.4] - 2026-07-22 ### Miscellaneous Tasks diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 4bd31772..1fccbb72 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-storage" -version = "0.2.4" +version = "0.2.6" description = "Storage-layer internals shared by S2 server implementations" edition.workspace = true license.workspace = true diff --git a/testcontainers/CHANGELOG.md b/testcontainers/CHANGELOG.md index aee84165..6d8b7d6d 100644 --- a/testcontainers/CHANGELOG.md +++ b/testcontainers/CHANGELOG.md @@ -2,6 +2,38 @@ All notable changes to this project will be documented in this file. +## [0.43.0] - 2026-09-25 + + + +## [0.42.14] - 2026-09-24 + + + +## [0.42.13] - 2026-09-22 + +### Miscellaneous Tasks + +- Update Cargo.toml dependencies + + + +## [0.42.12] - 2026-09-16 + + + +## [0.42.11] - 2026-09-11 + + + +## [0.42.10] - 2026-09-11 + + + +## [0.42.9] - 2026-09-10 + + + ## [0.42.8] - 2026-09-01 diff --git a/testcontainers/Cargo.toml b/testcontainers/Cargo.toml index 3dfb58e1..23647de1 100644 --- a/testcontainers/Cargo.toml +++ b/testcontainers/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s2-testcontainers" -version = "0.42.8" +version = "0.43.0" description = "Testcontainers helpers for the S2 Docker image" edition.workspace = true license.workspace = true