diff --git a/.codecov.yml b/.codecov.yml index 120f9c5626d..2f3cfb79b6a 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -132,6 +132,11 @@ ignore: - "packages/rs-drive-abci/src/mimic/**" - "packages/rs-sdk-trusted-context-provider/**" - "packages/rs-drive-abci/src/replay/**" + # Shielded-pool snapshot bake/apply tooling — only reachable via the + # `snapshot-bake` CLI subcommand (build-time) and the InitChain + # `DRIVE_SHIELDED_SNAPSHOT` env fast-path; both require full GroveDB genesis + # seeding and are exercised by create_genesis_state integration tests + - "packages/rs-drive-abci/src/shielded_snapshot/**" # DPP signing test module — integration tests, not unit-testable - "packages/rs-dpp/src/state_transition/state_transitions/address_funds/**/signing_tests.rs" # Drive extra tests — integration tests in production code tree diff --git a/Cargo.lock b/Cargo.lock index 16b6b5700d6..d778bf17520 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2018,6 +2018,7 @@ dependencies = [ "assert_matches", "async-trait", "bincode", + "blake3", "bls-signatures", "chrono", "ciborium", @@ -2035,6 +2036,8 @@ dependencies = [ "envy", "file-rotate", "grovedb-commitment-tree", + "grovedb-path", + "grovedb-storage", "hex", "indexmap 2.14.0", "integer-encoding", @@ -2686,7 +2689,7 @@ dependencies = [ [[package]] name = "grovedb" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" +source = "git+https://github.com/dashpay/grovedb?rev=5eb7a5380a6e974513343352acfd6b30a8c1f87c#5eb7a5380a6e974513343352acfd6b30a8c1f87c" dependencies = [ "axum 0.8.9", "bincode", @@ -2724,7 +2727,7 @@ dependencies = [ [[package]] name = "grovedb-bulk-append-tree" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" +source = "git+https://github.com/dashpay/grovedb?rev=5eb7a5380a6e974513343352acfd6b30a8c1f87c#5eb7a5380a6e974513343352acfd6b30a8c1f87c" dependencies = [ "bincode", "blake3", @@ -2740,7 +2743,7 @@ dependencies = [ [[package]] name = "grovedb-commitment-tree" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" +source = "git+https://github.com/dashpay/grovedb?rev=5eb7a5380a6e974513343352acfd6b30a8c1f87c#5eb7a5380a6e974513343352acfd6b30a8c1f87c" dependencies = [ "blake3", "grovedb-bulk-append-tree", @@ -2756,7 +2759,7 @@ dependencies = [ [[package]] name = "grovedb-costs" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" +source = "git+https://github.com/dashpay/grovedb?rev=5eb7a5380a6e974513343352acfd6b30a8c1f87c#5eb7a5380a6e974513343352acfd6b30a8c1f87c" dependencies = [ "integer-encoding", "intmap", @@ -2766,7 +2769,7 @@ dependencies = [ [[package]] name = "grovedb-dense-fixed-sized-merkle-tree" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" +source = "git+https://github.com/dashpay/grovedb?rev=5eb7a5380a6e974513343352acfd6b30a8c1f87c#5eb7a5380a6e974513343352acfd6b30a8c1f87c" dependencies = [ "bincode", "blake3", @@ -2779,7 +2782,7 @@ dependencies = [ [[package]] name = "grovedb-element" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" +source = "git+https://github.com/dashpay/grovedb?rev=5eb7a5380a6e974513343352acfd6b30a8c1f87c#5eb7a5380a6e974513343352acfd6b30a8c1f87c" dependencies = [ "bincode", "bincode_derive", @@ -2794,7 +2797,7 @@ dependencies = [ [[package]] name = "grovedb-epoch-based-storage-flags" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" +source = "git+https://github.com/dashpay/grovedb?rev=5eb7a5380a6e974513343352acfd6b30a8c1f87c#5eb7a5380a6e974513343352acfd6b30a8c1f87c" dependencies = [ "grovedb-costs", "hex", @@ -2806,7 +2809,7 @@ dependencies = [ [[package]] name = "grovedb-merk" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" +source = "git+https://github.com/dashpay/grovedb?rev=5eb7a5380a6e974513343352acfd6b30a8c1f87c#5eb7a5380a6e974513343352acfd6b30a8c1f87c" dependencies = [ "bincode", "bincode_derive", @@ -2832,7 +2835,7 @@ dependencies = [ [[package]] name = "grovedb-merkle-mountain-range" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" +source = "git+https://github.com/dashpay/grovedb?rev=5eb7a5380a6e974513343352acfd6b30a8c1f87c#5eb7a5380a6e974513343352acfd6b30a8c1f87c" dependencies = [ "bincode", "blake3", @@ -2843,7 +2846,7 @@ dependencies = [ [[package]] name = "grovedb-path" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" +source = "git+https://github.com/dashpay/grovedb?rev=5eb7a5380a6e974513343352acfd6b30a8c1f87c#5eb7a5380a6e974513343352acfd6b30a8c1f87c" dependencies = [ "hex", ] @@ -2851,7 +2854,7 @@ dependencies = [ [[package]] name = "grovedb-query" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" +source = "git+https://github.com/dashpay/grovedb?rev=5eb7a5380a6e974513343352acfd6b30a8c1f87c#5eb7a5380a6e974513343352acfd6b30a8c1f87c" dependencies = [ "bincode", "byteorder", @@ -2867,7 +2870,7 @@ dependencies = [ [[package]] name = "grovedb-storage" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" +source = "git+https://github.com/dashpay/grovedb?rev=5eb7a5380a6e974513343352acfd6b30a8c1f87c#5eb7a5380a6e974513343352acfd6b30a8c1f87c" dependencies = [ "blake3", "grovedb-costs", @@ -2886,7 +2889,7 @@ dependencies = [ [[package]] name = "grovedb-version" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" +source = "git+https://github.com/dashpay/grovedb?rev=5eb7a5380a6e974513343352acfd6b30a8c1f87c#5eb7a5380a6e974513343352acfd6b30a8c1f87c" dependencies = [ "thiserror 2.0.18", "versioned-feature-core 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2895,7 +2898,7 @@ dependencies = [ [[package]] name = "grovedb-visualize" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" +source = "git+https://github.com/dashpay/grovedb?rev=5eb7a5380a6e974513343352acfd6b30a8c1f87c#5eb7a5380a6e974513343352acfd6b30a8c1f87c" dependencies = [ "hex", "itertools 0.14.0", @@ -2904,7 +2907,7 @@ dependencies = [ [[package]] name = "grovedbg-types" version = "4.0.0" -source = "git+https://github.com/dashpay/grovedb?rev=60f29685172653f6007e63d0916bce4633bc23b9#60f29685172653f6007e63d0916bce4633bc23b9" +source = "git+https://github.com/dashpay/grovedb?rev=5eb7a5380a6e974513343352acfd6b30a8c1f87c#5eb7a5380a6e974513343352acfd6b30a8c1f87c" dependencies = [ "serde", "serde_with 3.20.0", @@ -4829,6 +4832,8 @@ dependencies = [ "dash-spv", "dashcore", "dpp", + "drive-proof-verifier", + "futures", "grovedb-commitment-tree", "hex", "image", @@ -4836,6 +4841,9 @@ dependencies = [ "key-wallet-manager", "platform-encryption", "rand 0.8.6", + "rayon", + "rs-sdk-trusted-context-provider", + "rusqlite", "serde", "serde_json", "sha2", diff --git a/Dockerfile b/Dockerfile index d4c787b7fc3..ff14ceaf1d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,8 +18,12 @@ # - ALPINE_VERSION - use different version of Alpine base image; requires also rust:apline... # image to be available # - USERNAME, USER_UID, USER_GID - specification of user used to run the binary -# - SDK_TEST_DATA - set to `true` to create SDK test data on chain genesis. It should be used only for testing -# purpose in local development environment +# - SDK_TEST_DATA - set to `true` to create SDK test data on chain genesis. +# For local devnet workflows use `yarn dashmate config set +# platform.drive.abci.docker.build.buildArgs.SDK_TEST_DATA true` (the +# `yarn setup` script does this automatically for the `local` config) — +# do NOT pass it as a shell env. The value flows through dashmate -> +# docker-compose `build.args:` -> this ARG. # # # sccache cache backends # @@ -422,9 +426,11 @@ RUN --mount=type=secret,id=AWS \ # This will prebuild majority of dependencies FROM deps AS build-drive-abci -# Pass SDK_TEST_DATA=true to create SDK test data on chain genesis -# This is only for testing purpose and should be used only for -# local development environment +# SDK_TEST_DATA is forwarded by dashmate from each `local_N` config's +# `platform.drive.abci.docker.build.buildArgs.SDK_TEST_DATA` field (set by +# `scripts/setup_local_network.sh` after `dashmate setup local`, as part of +# `yarn setup`). Do NOT set this via shell env — single source of truth is the +# dashmate config. ARG SDK_TEST_DATA ARG ADDITIONAL_FEATURES="" @@ -554,6 +560,37 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM rm -rf /platform +# +# STAGE: BAKE SHIELDED-POOL SNAPSHOT +# +# Self-contained bake step: runs `drive-abci snapshot-bake` against a fresh +# in-container tempdir to produce /artifacts/shielded-pool.snap. The runtime +# image COPYs that file in and sets `DRIVE_SHIELDED_SNAPSHOT` so the +# InitChain hook applies it instead of running the runtime seeder. +# +# Skipped (file replaced with a sentinel) when SDK_TEST_DATA != "true", so +# production / non-SDK builds don't carry test data. +# +FROM build-drive-abci AS bake-shielded-snapshot + +ARG SDK_TEST_DATA + +# libgcc + libstdc++ for the dynamically-linked drive-abci binary (build +# stage's alpine image normally has them; explicit `apk add` is a no-op if +# already present). +RUN apk add --no-cache libgcc libstdc++ + +RUN set -ex; \ + mkdir -p /artifacts; \ + if [ "${SDK_TEST_DATA}" = "true" ]; then \ + /artifacts/drive-abci snapshot-bake --out /artifacts/shielded-pool.snap ; \ + ls -la /artifacts/shielded-pool.snap ; \ + else \ + echo "SDK_TEST_DATA != true; skipping shielded-pool snapshot bake" ; \ + : > /artifacts/.no-shielded-snapshot ; \ + fi + + # # STAGE: BUILD JAVASCRIPT INTERMEDIATE IMAGE # @@ -667,8 +704,22 @@ RUN mkdir -p /var/log/dash \ ${REJECTIONS_PATH} COPY --from=build-drive-abci /artifacts/drive-abci /usr/bin/drive-abci +COPY --from=bake-shielded-snapshot /artifacts/ /opt/dashmate/snapshots/ COPY packages/rs-drive-abci/.env.mainnet /var/lib/dash/rs-drive-abci/.env +# Only point InitChain's apply-side at the snapshot when the bake stage +# actually produced one (SDK_TEST_DATA=true). On the SDK_TEST_DATA=false +# branch the bake stage leaves only a `.no-shielded-snapshot` sentinel, so +# exporting DRIVE_SHIELDED_SNAPSHOT unconditionally would make +# create_data_for_shielded_pool try to apply a missing file and fail +# instead of falling back to the runtime seeder. We gate on the real file's +# existence (writing the var into the binary's .env, which is loaded via +# dotenvy and left unset otherwise so the seeder fallback runs). +RUN if [ -f /opt/dashmate/snapshots/shielded-pool.snap ]; then \ + echo "DRIVE_SHIELDED_SNAPSHOT=/opt/dashmate/snapshots/shielded-pool.snap" \ + >> /var/lib/dash/rs-drive-abci/.env ; \ + fi + # Create a volume VOLUME /var/lib/dash/rs-drive-abci/db VOLUME /var/log/dash diff --git a/docs/genesis-snapshot-design.md b/docs/genesis-snapshot-design.md new file mode 100644 index 00000000000..9b8c44f068a --- /dev/null +++ b/docs/genesis-snapshot-design.md @@ -0,0 +1,723 @@ +# Shielded-pool genesis snapshot — design + +Pivots PR #3732 away from runtime seeding into a **shielded-pool-specific +snapshot bake + RocksDB ingest** model. One-shot tool, no future-proof +genericity — every Element kind that ever needs this gets its own dedicated +path (YAGNI until a second case appears). + +## 1. Problem statement + +The Option-A path that the current PR implements (run the seeder at +`InitChain`) takes **~3h 41m for 500_000 notes on macOS Docker** — release +profile, single attempt, no retries. CPU work is ~100 s; the remaining +~3h 40m is GroveDB write amplification through Docker Desktop's file-share +layer. Unusable for any reset-driven dev loop. + +The Sinsemilla math is fundamentally sequential per-tree (each append +depends on prior frontier state), so per-insert hot-path optimisations cap +at single-digit minutes. The only way to drop the cost meaningfully is to +**avoid doing the 500k writes at runtime**. + +## 2. Approach + +Move the writes to an **offline bake** that runs inside docker buildkit's +linux VM (fast native fsync), serialise the resulting subtree's RocksDB +state into a portable snapshot file, ship that file in the image, and have +`InitChain` `ingest_external_file_cf` the SST contents back into the live DB +at boot. After ingest, **replay the cmx stream to recompute the two roots +(`bulk_state_root` + `sinsemilla_root`) and cross-validate** against the +header's values — any tampering or version skew surfaces here, before the +parent Merk is touched. Then `Element::insert_subtree` writes the parent +Merk leaf and Merk propagation runs normally. + +Three components: + +``` +┌──────────────────────────────────────┐ ┌───────────────────────────┐ +│ snapshot-bake CLI (one-shot) │ │ InitChain (every reset) │ +│ rs-drive-abci src/bin/ │ │ rs-drive-abci genesis │ +│ snapshot-bake.rs │ │ path │ +│ │ │ │ +│ 1. open tmp grovedb in /tmp │ │ 1. create_genesis_state │ +│ 2. seed_shielded_pool_with_config() │ │ builds parent tree │ +│ 3. dump_shielded_subtree(...) │ │ 2. apply_shielded_ │ +│ │ │ snapshot_if_set(...) │ +│ │ │ (inside the same txn) │ +└──────────────────────────────────────┘ └───────────────────────────┘ + │ ▲ + ▼ │ + ┌──────────────────────────────────────────────────┐ + │ rs-drive-abci::shielded_snapshot module │ + │ - dump_shielded_subtree(grove, w) -> Stats │ + │ - apply_shielded_snapshot(grove, r, txn) -> () │ + │ - shielded-specific. No dispatch table. │ + └──────────────────────────────────────────────────┘ +``` + +Both `dump` and `apply` live in `rs-drive-abci` (not `rs-drive`) because they +co-locate with the seeder and the existing shielded subtree code. No new +public surface in `rs-drive`. The grovedb-storage surface additions (per-CF +iteration, raw-db access) live in `grovedb-storage` itself because they're +foundational. + +## 3. Snapshot file format + +Single self-describing binary file, shielded-specific. The two crypto roots +are **carried in the header AND independently recomputed at apply time** — +the header values are a hint the apply path cross-validates against, not a +source of truth. + +``` ++──────────────────+──────+─────────────────────────────────────────────+ +| Field | Size | Description | ++──────────────────+──────+─────────────────────────────────────────────+ +| magic | 8 B | "DRVSHLD\0" | +| format_version | u32 | 1 (this revision). Bump = breaking change. | +| grovedb_git_sha | 20 B | git SHA of grovedb the bake ran against. Hard| +| | | error if mismatched with the runtime build. | +| total_count | u64 | number of cmx commitments in the subtree | +| chunk_power | u8 | BulkAppendTree chunk_power (≤ 16 — enforced | +| | | at apply time as DoS sanity) | +| flags | u8 | Element::CommitmentTree flags byte | +| sinsemilla_root | 32 B | Pallas-base — header hint, cross-validated | +| | | at apply time by replaying the cmx stream. | +| bulk_state_root | 32 B | blake3("bulk_state" || mmr_root || buffer | +| | | hash) — header hint, cross-validated. | +| default_sst_len | u64 | length of default-CF SST blob | +| default_sst | var | SST containing META_KEY, chunk blobs, MMR | +| | | nodes — produced by SstFileWriter | +| aux_sst_len | u64 | length of aux-CF SST blob | +| aux_sst | var | SST containing the Sinsemilla frontier | +| | | (`__ct_data__` key) | +| checksum | 32 B | blake3 over everything above | ++──────────────────+──────+─────────────────────────────────────────────+ +``` + +**CF routing is fixed and explicit.** Per feasibility review: +`DataBulkStore` (used by `commitment_tree_insert` line 249) calls +`ctx.put/get` which route to **default CF**. So META_KEY (`b"M"`), chunk +blobs (`e{u64}`), tail buffer (`b{u32}`), and chunk-MMR nodes (`m{u64}`) +**all live in default CF**. The Sinsemilla frontier +(`COMMITMENT_TREE_DATA_KEY = b"__ct_data__"`) is the only thing in **aux** +(written via `put_aux` at `commitment_tree.rs:288-291`). Dump captures both; +apply ingests both. + +**Why embedded SST bytes vs (k,v) tuples:** lets the bake emit final +RocksDB-ready bytes (no per-record fsync), and the apply side writes the +section to a tmp file and calls `ingest_external_file_cf` once per CF. + +## 4. Module API + +In `packages/rs-drive-abci/src/shielded_snapshot/mod.rs`: + +```rust +pub fn dump_shielded_subtree( + grove: &GroveDb, + transaction: TransactionArg, + out_path: &Path, + platform_version: &PlatformVersion, +) -> Result; + +pub fn apply_shielded_snapshot( + grove: &GroveDb, + transaction: TransactionArg, + snapshot_path: &Path, + platform_version: &PlatformVersion, +) -> Result; +``` + +Errors are structured. `ShieldedSnapshotError::PartiallyApplied { ingested_cfs, failed_cf, cause }` lets the caller distinguish "no-op, +retry" from "DB in partial state, wipe and re-bootstrap." + +**`dump_shielded_subtree`** is shielded-specific. The subtree path is +hardcoded as the well-known shielded pool location. The dump iterates the +subtree's prefix in both default and aux CFs, emitting two `SstFileWriter` +streams. Reads the parent-Merk leaf to capture `total_count`, `chunk_power`, +`flags`, `sinsemilla_root`, `bulk_state_root` for the header. + +**`apply_shielded_snapshot`** does, in order: + +1. Read + validate header (magic, `format_version == 1`, `grovedb_git_sha == + runtime git SHA`, `chunk_power ≤ 16`, checksum). +2. Extract each section's SST bytes to a tmp file. +3. Call `db.ingest_external_file_cf` on default CF, then aux CF, with + `IngestExternalFileOptions { allow_global_seqno: false, write_global_seqno: false, ..Default::default() }`. (Explicit per crypto review + D2 — global seqno injection would let a malicious snapshot poison + sequence ordering.) +4. Reconstruct `BulkAppendTree::load_from_store(store, total_count, chunk_power)`. Reads META_KEY etc from default CF. +5. Read the Sinsemilla frontier from aux CF, deserialize. +6. **Cross-validate** (per crypto F1/D1): + - Recompute `bulk_state_root` from the loaded BulkAppendTree's + `mmr_root` + `buffer_hash`. Compare to header. Mismatch → fatal. + - Walk the frontier's recorded leaves (or replay cmx → fresh frontier), + compare resulting `sinsemilla_root` to header. Mismatch → fatal. +7. Build `Element::CommitmentTree(sinsemilla_root, total_count, chunk_power, flags)` and call `element.insert_subtree(&mut parent_merk, key, bulk_state_root, txn)`. +8. `propagate_changes_with_transaction(txn)`. + +Step 6 is the non-negotiable fix from crypto review — without it, a +corrupted snapshot could ingest a poisoned root that the chain then blesses. + +## 5. Bake tool + +`packages/rs-drive-abci/src/bin/snapshot-bake.rs`. CLI: + +``` +snapshot-bake shielded-pool --out +``` + +No dispatch table, no subcommand registry. Single hardcoded operation: open +tmp grovedb, run the existing +`seed_shielded_pool_with_config(&ShieldedSeedConfig::sdk_test_data())`, call +`dump_shielded_subtree(grove, &mut file)`. Per scope review. + +The bake binary is built with the **same cfg flags as the runtime +drive-abci** (specifically `create_sdk_test_data`). Per crypto review D5 — +mismatched cfg would let the bake produce a shape the runtime can't accept. +Dockerfile enforces this by deriving both from the same builder stage. + +`DEFAULT_OPTS` (RocksDB Options) is shared between bake and apply paths via +a `pub fn snapshot_db_options() -> Options` in `grovedb-storage` so the SST +files are guaranteed compatible. + +## 6. InitChain integration + +In `packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/mod.rs`, +**inside the existing InitChain transaction**, after the parent tree +structure (`/platform/`, `/platform/shielded_pool/`, ...) is built but +before `create_data_for_shielded_pool` would have run: + +```rust +if let Ok(snapshot_path) = std::env::var("DRIVE_SHIELDED_SNAPSHOT") { + let mut file = File::open(&snapshot_path) + .map_err(|e| Error::Execution(format!( + "DRIVE_SHIELDED_SNAPSHOT={snapshot_path} unreadable: {e}" + ).into()))?; + shielded_snapshot::apply_shielded_snapshot( + &self.drive.grove, + &mut file, + transaction, + ).map_err(Error::Execution)?; +} +``` + +Transactional contract (per feasibility F3 + your direction): + +- The ingest_external_file_cf calls **bypass the transaction layer** — they + write SSTs directly to default+aux CFs at the DB level. +- The parent-Merk leaf write (`element.insert_subtree`) **goes through the + transaction** like any other Merk op. +- Failure mode: if the InitChain transaction aborts AFTER ingest but BEFORE + commit, the default+aux data persists with no parent leaf pointing at it. + This is an orphan, but **InitChain abort already requires wipe-and-restart + recovery** (a half-bootstrapped chain isn't usable), so the orphan + resolves naturally. +- The cmx-replay cross-validation in step 6 of §4 happens **before** the + parent leaf write. If validation fails, the transaction proceeds with no + parent leaf and the InitChain handler returns an error, triggering + exactly the wipe-and-restart path the abort case relies on. + +`create_data_for_shielded_pool` branches on the `DRIVE_SHIELDED_SNAPSHOT` +env var: + +- **Snapshot fast-path (env var set):** the snapshot file is applied via SST + ingest. Any failure here — file missing, magic mismatch, format/version + skew, checksum failure, or `combined_root` cross-validation drift — is + **fatal**: `apply_shielded_snapshot` returns a `ShieldedSnapshotError` and + InitChain surfaces it as `Error::Execution(...)` and aborts. There is no + silent fallback to the seeder once the env var is set; the operator is + expected to fix the snapshot rather than get different state than asked for. +- **Runtime seeding fallback (env var unset):** the chain seeds the shielded + pool at genesis from `ShieldedSeedConfig::sdk_test_data()` (the slow + Sinsemilla runtime path). This is the path used when the binary is built + with `create_sdk_test_data` but no baked snapshot is shipped. + +The Dockerfile only exports `DRIVE_SHIELDED_SNAPSHOT` when the bake stage +actually produced a snapshot file (`SDK_TEST_DATA=true`); otherwise the bake +stage leaves a `.no-shielded-snapshot` sentinel and the env var is left unset +so the runtime seeder fallback runs. The bake-then-apply route is the fast, +recommended way to populate the shielded pool at genesis. + +`record_shielded_pool_anchor_if_changed(height=1)` runs after the snapshot +apply, same as before, so the anchor matches the snapshot's frontier. + +## 7. Dockerfile + dashmate plumbing + +Bake stage runs in the linux VM where fsync is fast: + +```dockerfile +# After build-drive-abci, gated on SDK_TEST_DATA build arg: +FROM build-drive-abci AS bake-shielded-snapshot +ARG SDK_TEST_DATA=false +RUN if [ "$SDK_TEST_DATA" = "true" ]; then \ + cargo build --release --bin snapshot-bake \ + --features create_sdk_test_data && \ + target/release/snapshot-bake shielded-pool \ + --out /artifacts/shielded-pool.snap ; \ + else \ + mkdir -p /artifacts && \ + : > /artifacts/shielded-pool.snap.absent ; \ + fi + +FROM AS drive-abci +COPY --from=build-drive-abci /artifacts/drive-abci /usr/bin/drive-abci +COPY --from=bake-shielded-snapshot /artifacts/ /opt/dashmate/snapshots/ +``` + +`SDK_TEST_DATA=true` is the existing dashmate build arg +(`platform.drive.abci.docker.build.buildArgs.SDK_TEST_DATA`). Per crypto D5, +the bake binary is compiled with `--features create_sdk_test_data` so it +matches the runtime's cfg view of `Element`/seeder shape. + +Per-platform bake under `buildx` for cross-arch (arm64 mac vs amd64 +production). SST files embed compression block decisions made by +`SstFileWriter` — these must match the apply-side `Options`, which is why +both sides pull from `grovedb_storage::snapshot_db_options()`. + +Dashmate side: drive-abci container env adds + +```yaml +environment: + - DRIVE_SHIELDED_SNAPSHOT=${DRIVE_SHIELDED_SNAPSHOT:-/opt/dashmate/snapshots/shielded-pool.snap} +``` + +Default points at the in-image snapshot. Empty string or unset = no apply +(genesis runs without shielded data). Operators wanting a custom snapshot +mount-bind a different file and set the env. + +**Removed in this PR:** +- `CARGO_BUILD_PROFILE=release` workaround in `scripts/setup_local_network.sh` (no longer needed — release profile speedup + was for the runtime seeder, which is gone). +- `create_data_for_shielded_pool` runtime path. +- Slow-seeder fallback option. Hard requirement on the snapshot file when + `create_sdk_test_data` is built in. + +## 8. Correctness reasoning (Phase 1 only — see §15 for Phase 2 deltas) + +> **Scope:** this section's "byte-identical to what the runtime would +> have written" claim holds for the Phase 1 design (full-Sinsemilla +> seed in the bake). Phase 2 (§15) deliberately drops that property for +> the speedup. After Phase 2 lands, the bake produces a snapshot whose +> `combined_root` is self-consistent but **not equal** to what a +> runtime-seeded chain would produce — see §15.5. + +**Why bulk-loaded data + cross-validation + Merk propagation produces the +same root as runtime seeding:** + +1. The bake tool runs the **exact same `commitment_tree_insert_op` sequence** + the runtime would. The resulting key-value pairs in storage are + byte-identical to what the runtime would have written. Same RNG seed + (`ShieldedSeedConfig::sdk_test_data().rng_seed = 0xDEAD_BEEF`), same note + generator path, same Sinsemilla state machine. + +2. `dump_shielded_subtree` enumerates those key-value pairs from the tmp DB + (default CF: META_KEY + chunk blobs + buffer + MMR nodes; aux CF: + Sinsemilla frontier) and writes them into SST files. RocksDB SST format + preserves key+value bytes byte-for-byte. + +3. `apply_shielded_snapshot` ingests both SSTs. RocksDB's bulk-load path + writes to the LSM tree without going through WAL, but post-ingest the + key-value visibility is identical to having done individual `put` calls. + `IngestExternalFileOptions { allow_global_seqno: false }` prevents + sequence-number injection that could let a poisoned snapshot reorder + later writes. + +4. **Cross-validation (step 6 of §4) is what makes this safe.** We don't + trust the header's `bulk_state_root` and `sinsemilla_root`. We reload + the BulkAppendTree from the freshly-ingested storage, recompute + `bulk_state_root = blake3("bulk_state" || mmr_root || buffer_hash)`, and + replay the cmx stream through a fresh Sinsemilla frontier to recompute + `sinsemilla_root`. Both must equal the header values byte-for-byte. If + anyone tampered with the SST data, or if grovedb's storage layout + changed between bake and apply, the recomputed roots diverge and we + fail before touching the parent Merk. + +5. `element.insert_subtree(parent, key, bulk_state_root, txn)` writes the + parent Merk leaf with the validated `bulk_state_root` as the child hash, + inside the InitChain transaction. + +6. `propagate_changes_with_transaction(txn)` runs the same hash propagation + the runtime would. After commit, every ancestor Merk root matches what + the slow seeder would have produced. + +7. A regression test (§10 #6) bakes the snapshot, applies to chain A, runs + the runtime seeder on chain B against a tmp grovedb, asserts + `grove_a.root_hash() == grove_b.root_hash()` byte-for-byte. + +**Threat model (per crypto D4):** + +- **In scope:** corrupted/truncated SST bytes (cross-validation + RocksDB + ingest validation catches), header tampering with roots (cross-validation + catches), grovedb version skew (git SHA hard error). +- **Out of scope by design:** cross-devnet snapshot replay (devnets share + the same shielded pool layout — that's the point), image-supply-chain + trust (we trust the docker image's bake stage produced honest bytes; this + is the same trust boundary as the rest of the image). +- **Not addressed:** snapshot signing. Devnet-only; signing adds key-mgmt + surface for no current threat. Decision deferred. If we ever ship this + for testnet/mainnet, revisit. + +## 9. Edge cases & error handling + +| Case | Behaviour | +|---|---| +| `DRIVE_SHIELDED_SNAPSHOT` unset, `create_sdk_test_data` NOT built | No-op. Chain boots normally with empty shielded pool. | +| `DRIVE_SHIELDED_SNAPSHOT` unset, `create_sdk_test_data` built | **Fatal**: `Error::Execution("DRIVE_SHIELDED_SNAPSHOT required when built with create_sdk_test_data")`. Forces the bake stage to be wired correctly. | +| File doesn't exist | Fatal: `Error::Execution(SnapshotFileNotFound { path })` | +| Magic mismatch | Fatal: `ShieldedSnapshotError::InvalidMagic` | +| `format_version` mismatch | Fatal: `ShieldedSnapshotError::FormatVersionMismatch { expected, found }` | +| `grovedb_git_sha` mismatch | Fatal: `ShieldedSnapshotError::GrovedbRevMismatch { expected, found }` | +| `chunk_power > 16` | Fatal: `ShieldedSnapshotError::ChunkPowerTooLarge { got, max: 16 }` | +| Checksum mismatch | Fatal: `ShieldedSnapshotError::Corrupted` | +| RocksDB ingest fails (CF overlap, etc.) | Fatal: `ShieldedSnapshotError::IngestFailed { cf, cause }` | +| `BulkAppendTree::load_from_store` fails | Fatal: `ShieldedSnapshotError::CorruptedAfterIngest { cause }` — at this point the default CF data is on disk; caller should treat as PartiallyApplied. | +| `bulk_state_root` cross-validation mismatch | Fatal: `ShieldedSnapshotError::BulkStateRootMismatch { expected, computed }` | +| `sinsemilla_root` cross-validation mismatch | Fatal: `ShieldedSnapshotError::SinsemillaRootMismatch { expected, computed }` | +| Parent subtree (`/platform/shielded_pool/`) missing in target DB | Fatal: `ShieldedSnapshotError::ParentSubtreeMissing` — snapshot must run after `create_genesis_state` builds the parent tree. | + +## 10. Testing strategy + +Tests live in `packages/rs-drive-abci/tests/shielded_snapshot/`: + +1. **Roundtrip** — small (N=16) shielded subtree: + - Build via normal grovedb ops (existing `seed_shielded_pool_with_config`) + - `dump_shielded_subtree` to a buffer + - `apply_shielded_snapshot` into a fresh DB (with parent tree pre-built) + - Assert grovedb root hash matches between source and target + - Assert recorded anchor matches +2. **Same-host determinism** (per crypto D6) — bake the shielded subtree + twice on the same host with the same `RUSTC_BOOTSTRAP`/cfg flags; assert + snapshot file bytes are identical. Cross-host determinism is **not** + asserted (relies on RocksDB internals). +3. **Format-version refusal** — craft a snapshot with `format_version=2`; + assert `FormatVersionMismatch`. +4. **Git-SHA refusal** — craft a snapshot with a different `grovedb_git_sha`; + assert `GrovedbRevMismatch`. +5. **Corruption** — flip bits in default-CF SST, in aux-CF SST, in header + roots; assert `Corrupted` (checksum) or root-mismatch (cross-validation) + for each. +6. **Equivalence with runtime seeder** (RETIRED under Phase 2) — bake + snapshot, apply to chain A; run runtime seeder on chain B; assert + `grove_a.root_hash() == grove_b.root_hash()`. This test was valid + under Phase 1 (full-Sinsemilla bake) but **fails under Phase 2** + because the bake uses `append_many_without_frontier` for filler, + producing a different `combined_root` than a runtime-seeded chain. + Remove or `#[cfg]`-split when Phase 2 lands; replace with a + "bake-side determinism" test (same config → same snapshot bytes + across two bakes). +7. **Cross-validation catches header tampering** — bake snapshot, flip the + `bulk_state_root` field in the header, fix the outer checksum; assert + `BulkStateRootMismatch`. Same with `sinsemilla_root` → + `SinsemillaRootMismatch`. +8. **Fuzz** (per crypto D7) — `cargo-fuzz` target on `read_header`. Verify + no panics on arbitrary byte input. +9. **chunk_power bound** — craft snapshot with `chunk_power=17`; assert + `ChunkPowerTooLarge`. +10. **End-to-end (against snapshot-bake CLI)**: + - Bake the shielded subtree to a tmp file + - Open a fresh `TempPlatform`, run `create_genesis_state` with + `DRIVE_SHIELDED_SNAPSHOT` pointing at the tmp file + - Assert `shielded_pool_notes_count == 500_000` + - Assert recorded anchor at height 1 matches the snapshot's frontier +11. **Functional wallet sync** — existing + `wallet_a_recovers_deterministic_balance_via_manager_sync` works + unchanged against a snapshot-loaded chain. + +## 11. Open questions & risks + +| # | Risk | Mitigation | +|---|---|---| +| 1 | SST cross-arch portability (arm64 ↔ amd64). | Bake per-`--platform` under buildx. Image layers already platform-tagged. | +| 2 | grovedb may cache subtree metadata in memory; raw ingest could leave caches stale. | `apply_shielded_snapshot` is the FIRST grovedb op that touches the shielded subtree path in the InitChain txn. No reads happen before it. | +| 3 | `ingest_external_file_cf` interaction with `OptimisticTransactionDB`'s pending write set. | Accepted: ingest bypasses txn, parent-leaf write goes through txn, abort = wipe-and-restart. Documented in §6. | +| 4 | If someone re-runs InitChain on a non-fresh DB, ingest may collide with existing keys. | `IngestExternalFileOptions { allow_global_seqno: false }` + default `failed_move=true` causes overlap to error. Recovery = wipe DB and restart, which is the standard InitChain idempotency model. | +| 5 | Bake stage doubles image-build time even when `SDK_TEST_DATA=false`. | Gated on the build arg — `if [ "$SDK_TEST_DATA" = "true" ]` skips the cargo build entirely in production. | +| 6 | grovedb internal storage layout changes between bake and apply. | `grovedb_git_sha` hard error catches this. If you change grovedb, re-bake. | +| 7 | Snapshot file size (~150 MB at 500k notes). | Acceptable for a one-shot dev tool. Compressed inside the docker layer. | + +## 12. Compatibility policy + +**`format_version` bump policy:** +- Adding new fields to the header → `format_version` bump (new schema). +- Changing the meaning of an existing field → `format_version` bump. +- Changing how a field is computed (e.g. `bulk_state_root` derivation) → + `format_version` bump. +- Bumping `grovedb_git_sha` alone does NOT bump `format_version`. + +When `format_version` bumps, the apply side rejects older snapshots with +`FormatVersionMismatch`. There is no migration path — re-bake. + +**`grovedb_git_sha` policy:** +- Embedded via build script: `vergen-git2` or similar produces a `const GROVEDB_GIT_SHA: [u8; 20]` at the grovedb crate root, exposed as `pub const fn grovedb_git_sha() -> [u8; 20]`. +- Bake writes its compiled-in value. Apply compares against its compiled-in + value. Mismatch = `GrovedbRevMismatch`. +- Dirty-tree builds embed the SHA of HEAD (not the dirty state). This means + during local dev, a bake against uncommitted changes can apply against a + runtime built from those same uncommitted changes — acceptable, no + cross-validation gap because the cmx-replay still catches semantic + divergence. + +## 13. Phasing + +**This PR (single):** +- `grovedb-storage` additions: per-CF iteration on `StorageContext` (default + aux), `pub fn snapshot_db_options() -> Options`, and either a `pub fn raw_db()` accessor or a higher-level `pub fn apply_subtree_ingest()` helper on `GroveDb` (TBD during implementation — whichever exposes less surface). +- `grovedb` build-script for `grovedb_git_sha`. +- `rs-drive-abci::shielded_snapshot` module with `dump_shielded_subtree` + `apply_shielded_snapshot` + `read_header` + structured error enum. +- `snapshot-bake` CLI (one operation, hardcoded shielded-pool). +- InitChain hook reading `DRIVE_SHIELDED_SNAPSHOT` inside the existing transaction. +- Dockerfile bake stage gated on `SDK_TEST_DATA=true`. +- Dashmate `DRIVE_SHIELDED_SNAPSHOT` env-var forwarding. +- Remove `create_data_for_shielded_pool` runtime path entirely. +- Remove `CARGO_BUILD_PROFILE=release` workaround in `setup_local_network.sh`. +- All 11 tests above. Fuzz target. Equivalence test gated `#[ignore]`. + +**Out of scope (follow-ups, if ever needed):** +- Other Element kinds (no genericity built in — if a second case appears, factor out shared pieces THEN, informed by two real cases). +- Multi-subtree snapshots. +- Snapshot signing. +- Streaming bake (for N > 5M). + +## 14. Non-goals + +- Not redesigning the runtime shielded seeder. The existing generator stays + as the source of truth for how notes are produced; the snapshot is a + frozen byte-for-byte snapshot of its output. +- Not introducing a new GroveDB storage format. +- Not abstracting over storage backends. Bake + ingest is RocksDB-only. + +## 15. Phase 2 — frontier-less seeding for fast bake at large N + +### 15.1 Motivation + +Empirically measured bake times after the Phase-1 design landed: + +| N | Bake wall-clock (Docker macOS, release) | Notes | +|---|---|---| +| 5_000 | ~30 s | proven via in-process roundtrip + devnet e2e | +| 500_000 | ~3 h 41 m | original Option-A measurement; matches "65 min native macOS × 3.4× Docker tax" | +| **1_000_000** | **>8 h, did not complete** | killed; estimated 4-5 h was wrong, Docker virt overhead worse than modelled | + +The Sinsemilla frontier-append step is **75-85% of per-cmx cost** at all +N. For 1M notes, 999_992 of which are filler that no wallet owns, that's +~6-9 hours of Pallas math producing an anchor that nothing depends on. + +### 15.2 Insight (grovedb PR #751, "claude/eloquent-margulis-8369cd") + +Wallet sync verifies note membership via **BulkAppendTree chunk proofs** +authenticated by `bulk_state_root` (blake3-rooted), NOT via spend proofs +authenticated by the Sinsemilla anchor. So a chain whose anchor is +"junk" (e.g. reflects only the 8 owned cmx appends) still serves syncs +correctly — filler notes are recoverable, balances are recoverable, the +only thing missing is spend-proof authorization (which we never test). + +The PR adds a test-only opt-out from the Sinsemilla path: + +```rust +// On CommitmentTree (gated by `test-seeding-ct` feature in +// grovedb-commitment-tree; forwarded as `commitment_tree_test_seeding` +// from the grovedb crate). +pub fn append_raw_without_frontier(cmx, rho, payload) -> AppendResult; +pub fn append_many_without_frontier(iter: I) -> BulkSeedSummary; +``` + +Both skip the Sinsemilla append entirely. The blake3-rooted BulkAppendTree ++ MMR + dense buffer + chunk blobs still get written. The frontier +storage at `__ct_data__` is left untouched (or reflects only the +non-skipped owned appends). + +Additional perf win in the same PR: **MMR root cache** (commit +`7541294e`) — append becomes O(1) instead of O(N) for the MMR +incremental-update step. Stacks with the frontier skip. + +### 15.2a Hard constraint discovered in crypto review (F1) + +**`append_*_without_frontier` rejects a non-empty frontier.** PR #751 +commit `c530e59a` (`grovedb-commitment-tree/src/commitment_tree/mod.rs:520-531`) +returns `InvalidData("frontier-less seeding requires an empty frontier")` +if `self.frontier.tree_size() != 0`. + +This means we **cannot interleave** filler skip-path calls with owned +full-path calls. The bake order MUST be: + +1. Bulk-seed ALL filler positions first via `append_many_without_frontier` + while frontier is empty. +2. THEN loop the 8 owned through the normal full-Sinsemilla path + (`commitment_tree_insert_op` / `append_raw`). At this point bulk + positions [N-8, N) get appended; frontier picks up its first 8 cmx + at frontier-positions 0..7 (independent counter from bulk). + +**Consequence:** owned cmx live at bulk positions **[N-8, N)** — +contiguous tail — not striped across the tree the way the Phase 1 +seeder placed them via `OwnedLayout::compute`. This shifts the +canonical placement; `OwnedLayout` must be updated accordingly (see +§15.3). + +### 15.3 How we integrate + +**Cherry-pick all 6 commits** of PR #751 onto our +`feat/snapshot-apply-public-api` branch (files don't overlap — they +touch `grovedb-commitment-tree` + `grovedb-bulk-append-tree`; we touch +`grovedb-storage` + `grovedb` + `grovedb/operations/commitment_tree.rs`'s +TAIL only). + +**Feature name (corrected from earlier draft):** `test-seeding-ct` on +`grovedb-commitment-tree`, forwarded as `test-seeding-ct` on `grovedb` +itself (per commit `df492578` rename). Earlier sections referenced +`commitment_tree_test_seeding` — that name was dropped. + +**Compile-time binding (per A2 review finding):** the runtime +`cfg(create_sdk_test_data)` gate and the Cargo `test-seeding-ct` +feature MUST always be set together. Add a `compile_error!` guard in +rs-drive-abci that fires if one is set without the other. This +prevents a build matrix bug where the cfg is on but the feature is +off (or vice versa), silently falling back to the slow path or +breaking the build trail. + +**Refactor `OwnedLayout`** in +`packages/rs-drive-abci/.../create_genesis_state/test/shielded.rs`: + +- Change `OwnedLayout::compute(total, owned_count)` to place owned at + the **tail**: `wallet_a` at positions `[N - owned_count, N - + owned_count + count_a)`, `wallet_b` at positions `[N - count_b, N)`. +- Update all in-process platform_tests (`seeded_pool_count_matches_*`, + the address recovery tests at lines 595/701/722/789 per arch review) + to expect the new positions. +- `wallet_at(pos)` mapping inverted accordingly. + +**Update `seed_shielded_pool_with_config`** in the same file: + +1. Generate the `Vec` via the existing + `generate_notes_for_test_wallets` (unchanged — only positions move). +2. Partition into `(filler, owned)` slices by ownership. +3. Open a single `CommitmentTree` via grovedb's storage context. +4. Call `commitment_tree.append_many_without_frontier(filler_iter)` — + ALL filler in one bulk call. Frontier untouched. +5. Loop the 8 owned cmx through the existing + `commitment_tree_insert_op` per-note path so they hit the full + Sinsemilla append + parent-Merk leaf update. +6. **Post-bake assertion:** read back `commitment_tree_count()` and + assert it equals `cfg.total_notes` (catches silent truncation from + a mid-loop panic — F9 finding). + +**Filler cmx — DEFER skipping rejection sampling (F6 caveat).** PR +#751 accepts non-Pallas cmx, but our review flagged that wallet +shardtree may compute `MerkleHashOrchard::from_bytes(cmx)` over filler +positions during sync (`packages/rs-platform-wallet/.../sync.rs:317-320`) +and reject if cmx is not a canonical Pallas base. **Empirically verify +before skipping the rejection sampler** — write a one-off test that +constructs a non-Pallas 32-byte sequence, feeds it as a filler cmx, +and asserts wallet sync still completes. If it fails, keep the +rejection sampler. + +### 15.4 Expected bake wall-clock after integration + +Per-note costs in release on this hardware (extrapolated from PR #751's +own benchmark + our measurements): + +| Path | Cost per note | +|---|---| +| Old full append (Sinsemilla + BulkAppendTree) | ~7-8 ms | +| New `append_raw_without_frontier` (blake3 only) | ~1-2 μs | + +For N=1_000_000 with 8 owned + 999_992 filler: + +| Phase | Old | New | +|---|---|---| +| Filler appends | ~2.2 h | ~1-2 s | +| Owned appends | ~50 ms | ~50 ms | +| BulkAppendTree compactions (488 of them) | ~3 s | ~3 s | +| Frontier serialize/save | ~1 ms | ~1 ms | +| Dump (SST write) | ~5 s | ~5 s | +| **Total** | **~2.2 h** | **~10-30 s of CPU** | + +Add docker/disk overhead: **end-to-end bake stage ~5-10 min total** for +N=1M (mostly fixed cargo/init/SST-finalize overhead, not Sinsemilla +work). + +### 15.5 Consequences (must document in code + design) + +- **Anchor is not a valid Orchard anchor.** After the seeded chain + finalises InitChain, the recorded anchor at height 1 reflects only + the 8 owned cmx, sitting at **frontier positions 0..7** (regardless + of their bulk positions [N-8, N)). Production wallets attempting to + construct spend proofs against this anchor will fail because their + shardtree witness has cmx at bulk position N-8+k, but the on-chain + anchor only knows it at frontier-position k — `merkle_path.root(cmx) != anchor`. + This is acceptable because: + - Devnet wallets in this test only sync, not spend. + - The chain is gated by `cfg(create_sdk_test_data)` + `SDK_TEST_DATA=true` + at image build — never reaches production. + - **The Sinsemilla frontier and the BulkAppendTree maintain + independent position counters** (grovedb-commitment-tree's + `CommitmentFrontier::append` uses `incrementalmerkletree::Frontier`'s + internal `position()`, NOT bulk's `total_count`). Confirmed by + crypto review F2. +- **Add `tracing::warn!` at `record_shielded_pool_anchor_if_changed` + call-site** noting the recorded anchor is NOT a valid Orchard spend + anchor when the skip-path was used. (A6a finding.) +- **Wallet sync still works** because sync uses chunk proofs + authenticated by the grovedb root hash, which transitively + authenticates `bulk_state_root` (via the parent-Merk leaf's child + hash = `compute_commitment_tree_state_root(sinsemilla, bulk_state)`). + The wallet never compares `cmx` to the on-chain anchor — it appends + to its own shardtree and uses `merkle_path.root(cmx)` from that + local witness. Sync is anchor-agnostic. Confirmed by crypto review + F4 (cited evidence: `packages/rs-drive-proof-verifier/src/proof.rs:2555-2565`, + `packages/rs-platform-wallet/src/wallet/shielded/sync.rs:312-325`, + `operations.rs:612-680`). +- **Cross-host determinism: same-host only.** Same RNG seed, same + skip-path invocations, same blake3 output on the same host. Bake + re-runs across hosts may legitimately produce different + `__ct_data__` byte layouts (depends on MMR compaction timing, + intermediate Merk operations). Phase 2 inherits the Phase 1 + same-host-only determinism property — does not weaken it. +- **Bake-time vs apply-time symmetry holds.** The snapshot stores + whatever state the bake produced (including the "junk" anchor); apply + ingests it verbatim; the combined-root cross-validation passes + because both sides compute combined_root from the same loaded data. + Confirmed by crypto review F5. +- **`combined_root` of a Phase-2 bake ≠ `combined_root` of a + runtime-seeded chain** with the same config. This breaks the §10 + test #6 equivalence claim and the §8 "byte-identical to runtime" + reasoning. Both have been scoped to Phase 1 above. + +### 15.6 Test coverage to add + +- **Modify** existing in-process roundtrip + (`snapshot_dump_apply_preserves_anchor`) to assert anchor is + non-empty (still true with owned-only frontier) AND wallet sync logic + still finds the 8 owned notes through chunk proofs. **Drop the + `#[ignore]` tag** — Phase 2 bake at N=1M is fast enough for default + CI (A6e finding). +- **Retire** §10 test #6 (equivalence with runtime seeder). Replace + with a "bake-side determinism" test: same config → same SST bytes + across two bakes on the same host. +- **Add** F6-precondition test: feed a deliberately non-Pallas-canonical + 32-byte cmx as a filler, run wallet sync, assert it completes. + Gates the "skip rejection sampling for filler" optimization. If this + test fails, keep the rejection sampler. +- **Add** a test that **explicitly fails on attempted spend** against + the Phase-2-seeded chain. Makes the "not spendable" caveat + self-documenting and catches a regression where someone later tries + to enable spends on test chains. +- **Add** a post-bake assertion that `commitment_tree_count() == + cfg.total_notes` to catch silent truncation from a panic mid-bake + (F9 finding). +- **Benchmark test:** assert N=1_000_000 bake completes in <10 min + wall-clock in CI (or `#[ignore]` if CI hardware is slow). Tracks the + actual perf claim of Phase 2 against regressions. + +### 15.7 Devnet visibility — progress logging + +Independent of the perf fix: add periodic progress logs to +`seed_shielded_pool_with_config`. Currently the seeder is silent +between the "seeding ..." start and "dumping ..." end log lines, which +made the 8-hour blind wait extra painful. Emit every 30 s: + +``` +seed progress: appended 412480/1000000 (41.2%), elapsed 124s, rate 3326/s, ETA 176s +``` + +Keeps the bake observable even if some future change re-introduces a +slow loop somewhere. +- Not building a universal subtree-snapshot library. Shielded-specific by + design. diff --git a/packages/dashmate/configs/defaults/getBaseConfigFactory.js b/packages/dashmate/configs/defaults/getBaseConfigFactory.js index e21b2111e88..6e4c1d5fd02 100644 --- a/packages/dashmate/configs/defaults/getBaseConfigFactory.js +++ b/packages/dashmate/configs/defaults/getBaseConfigFactory.js @@ -268,6 +268,7 @@ export default function getBaseConfigFactory() { context: path.join(PACKAGE_ROOT_DIR, '..', '..'), dockerFile: path.join(PACKAGE_ROOT_DIR, '..', '..', 'Dockerfile'), target: 'rs-dapi', + buildArgs: {}, }, }, metrics: { @@ -293,6 +294,10 @@ export default function getBaseConfigFactory() { context: path.join(PACKAGE_ROOT_DIR, '..', '..'), dockerFile: path.join(PACKAGE_ROOT_DIR, '..', '..', 'Dockerfile'), target: 'drive-abci', + // Extra docker build args (see `dockerBuild` schema). Common + // override: `CARGO_BUILD_PROFILE: "release"` for SDK_TEST_DATA + // shielded seeding at N > a few thousand. + buildArgs: {}, }, }, logs: { diff --git a/packages/dashmate/configs/getConfigFileMigrationsFactory.js b/packages/dashmate/configs/getConfigFileMigrationsFactory.js index adaa072685d..ddeb508bb23 100644 --- a/packages/dashmate/configs/getConfigFileMigrationsFactory.js +++ b/packages/dashmate/configs/getConfigFileMigrationsFactory.js @@ -1433,6 +1433,19 @@ export default function getConfigFileMigrationsFactory(homeDir, defaultConfigs) .get('platform.drive.tenderdash.docker.image'); } + // Backfill docker build `buildArgs` added to the base config. + // Pre-3.1.0 configs predate the field, so default it to an empty + // object when missing (idempotent: existing values are preserved). + if (options.platform?.drive?.abci?.docker?.build + && typeof options.platform.drive.abci.docker.build.buildArgs === 'undefined') { + options.platform.drive.abci.docker.build.buildArgs = {}; + } + + if (options.platform?.dapi?.rsDapi?.docker?.build + && typeof options.platform.dapi.rsDapi.docker.build.buildArgs === 'undefined') { + options.platform.dapi.rsDapi.docker.build.buildArgs = {}; + } + if (options.platform?.drive?.tenderdash?.p2p && typeof options.platform.drive.tenderdash.p2p.allowlistOnly === 'undefined') { options.platform.drive.tenderdash.p2p.allowlistOnly = defaultConfig diff --git a/packages/dashmate/docker-compose.build.drive_abci.yml b/packages/dashmate/docker-compose.build.drive_abci.yml index 480f49eda2e..5295214ad5d 100644 --- a/packages/dashmate/docker-compose.build.drive_abci.yml +++ b/packages/dashmate/docker-compose.build.drive_abci.yml @@ -16,6 +16,11 @@ services: SCCACHE_REGION: ${SCCACHE_REGION} SCCACHE_S3_KEY_PREFIX: ${SCCACHE_S3_KEY_PREFIX} SDK_TEST_DATA: ${SDK_TEST_DATA} + # Forwarded by dashmate from + # `platform.drive.abci.docker.build.buildArgs.CARGO_BUILD_PROFILE` + # (overridable per-invocation via shell env). "release" is required + # for SDK_TEST_DATA shielded seeding at N > a few thousand. + CARGO_BUILD_PROFILE: ${CARGO_BUILD_PROFILE:-dev} secrets: - GITHUB_TOKEN cache_from: diff --git a/packages/dashmate/docker-compose.build.rs-dapi.yml b/packages/dashmate/docker-compose.build.rs-dapi.yml index 82192f13bcc..d36eb2001b6 100644 --- a/packages/dashmate/docker-compose.build.rs-dapi.yml +++ b/packages/dashmate/docker-compose.build.rs-dapi.yml @@ -15,6 +15,8 @@ services: SCCACHE_BUCKET: ${SCCACHE_BUCKET} SCCACHE_REGION: ${SCCACHE_REGION} SCCACHE_S3_KEY_PREFIX: ${SCCACHE_S3_KEY_PREFIX} + # See drive_abci compose for the buildArgs forwarding contract. + CARGO_BUILD_PROFILE: ${CARGO_BUILD_PROFILE:-dev} secrets: - GITHUB_TOKEN cache_from: diff --git a/packages/dashmate/docker-compose.shielded-snapshot.yml b/packages/dashmate/docker-compose.shielded-snapshot.yml new file mode 100644 index 00000000000..527920cf1f9 --- /dev/null +++ b/packages/dashmate/docker-compose.shielded-snapshot.yml @@ -0,0 +1,17 @@ +--- +# Docker-compose override that wires a baked shielded-pool snapshot into the +# drive-abci container. Applied by dashmate when +# `PLATFORM_DRIVE_ABCI_SHIELDED_SNAPSHOT_HOST_PATH` is set in the env it +# exports to docker-compose (we plumb that via `setup_local_network.sh` or a +# shell export before `yarn start`). +# +# At apply time the InitChain hook in `create_data_for_shielded_pool` reads +# `DRIVE_SHIELDED_SNAPSHOT` and takes the snapshot fast-path instead of +# running the runtime seeder. See packages/rs-drive-abci/src/shielded_snapshot. + +services: + drive_abci: + environment: + - DRIVE_SHIELDED_SNAPSHOT=/var/lib/dash/rs-drive-abci/snapshots/shielded-pool.snap + volumes: + - ${PLATFORM_DRIVE_ABCI_SHIELDED_SNAPSHOT_HOST_PATH}:/var/lib/dash/rs-drive-abci/snapshots/shielded-pool.snap:ro diff --git a/packages/dashmate/src/commands/config/set.js b/packages/dashmate/src/commands/config/set.js index aae44f9ef7b..4f6e27992ce 100644 --- a/packages/dashmate/src/commands/config/set.js +++ b/packages/dashmate/src/commands/config/set.js @@ -1,5 +1,7 @@ import { Args } from '@oclif/core'; import ConfigBaseCommand from '../../oclif/command/ConfigBaseCommand.js'; +import Config from '../../config/Config.js'; +import InvalidOptionPathError from '../../config/errors/InvalidOptionPathError.js'; export default class ConfigSetCommand extends ConfigBaseCommand { static description = `Set config option @@ -38,8 +40,16 @@ Sets a configuration option in the default config flags, config, ) { - // check for existence - config.get(optionPath); + // Validate the path against the schema, not against the currently-set + // value. `config.get(...)` would throw `InvalidOptionPathError` for any + // key inside a map-shaped property (e.g. `…buildArgs.SDK_TEST_DATA`) + // because the value doesn't exist yet — that gate is the wrong shape for + // schemas that use `additionalProperties: ` to model open maps. + // `Config.isSchemaPathAllowed` walks the schema and permits descent into + // both typed `properties` and `additionalProperties` value schemas. + if (!Config.isSchemaPathAllowed(optionPath)) { + throw new InvalidOptionPathError(optionPath); + } let value; diff --git a/packages/dashmate/src/config/Config.js b/packages/dashmate/src/config/Config.js index 2644bef6986..80dbb95b29c 100644 --- a/packages/dashmate/src/config/Config.js +++ b/packages/dashmate/src/config/Config.js @@ -45,6 +45,74 @@ export default class Config { return lodashGet(this.options, path) !== undefined; } + /** + * Check whether a path is reachable per the config JSON schema (regardless + * of whether a value is currently set there). + * + * Use this when checking the legality of a `set` to a path that doesn't yet + * have a value — notably under map-shaped properties whose schema uses + * `additionalProperties: ` (e.g. `…build.buildArgs.`), where + * `config.has(...)` will return `false` even though `config.set(...)` is + * semantically legal. + * + * `configJsonSchema` IS the per-config schema — the top-level + * `properties: { description, group, docker, core, platform, … }` describes + * one config entry. Walks it descending through: + * - `properties[segment]` (typed field), + * - `additionalProperties` (variable-key map, only when the value is a + * schema object — `additionalProperties: false` blocks the descent), + * - `$ref` references into `#/definitions/...`. + * + * @param {string} path - dot-separated option path (e.g. + * `'platform.drive.abci.docker.build.buildArgs.SDK_TEST_DATA'`). + * @return {boolean} true when the path is allowed by the schema. + */ + static isSchemaPathAllowed(path) { + if (typeof path !== 'string' || path.length === 0) return false; + + const resolveRef = (node) => { + if (!node || typeof node !== 'object') return node; + if (typeof node.$ref !== 'string') return node; + const ref = node.$ref; + if (!ref.startsWith('#/')) return null; + const segments = ref.slice(2).split('/'); + let resolved = configJsonSchema; + for (const seg of segments) { + if (!resolved || typeof resolved !== 'object') return null; + resolved = resolved[seg]; + } + return resolveRef(resolved); + }; + + let node = resolveRef(configJsonSchema); + if (!node) return false; + + for (const segment of path.split('.')) { + node = resolveRef(node); + if (!node || typeof node !== 'object') return false; + + // Typed property. + if (node.properties && Object.prototype.hasOwnProperty.call(node.properties, segment)) { + node = node.properties[segment]; + continue; + } + + // Map with a schema for extra keys — descend into the value schema. + if ( + node.additionalProperties + && typeof node.additionalProperties === 'object' + ) { + node = node.additionalProperties; + continue; + } + + // No match and no permissive additionalProperties — path not allowed. + return false; + } + + return true; + } + /** * Get config option * diff --git a/packages/dashmate/src/config/configJsonSchema.js b/packages/dashmate/src/config/configJsonSchema.js index ecf969bde64..9356cc38ca8 100644 --- a/packages/dashmate/src/config/configJsonSchema.js +++ b/packages/dashmate/src/config/configJsonSchema.js @@ -32,6 +32,24 @@ export default { target: { type: ['string', 'null'], }, + // Extra build args forwarded to `docker compose build` for this image. + // Each key becomes an env var with the same name, picked up by the + // `${KEY:-default}` substitutions in `docker-compose.build.*.yml` + // which mirror them into the Dockerfile `ARG`s. Shell env overrides + // any value set here (so `CARGO_BUILD_PROFILE=release yarn start` + // wins per-invocation). + // + // Common keys (when meaningful for the image's target): + // - CARGO_BUILD_PROFILE: "dev" | "release" — Rust profile for + // drive-abci / rs-dapi. Release is required for SDK_TEST_DATA + // shielded seeding at N > a few thousand. + // - SDK_TEST_DATA: "true" — enable the SDK test-data cfg flag in + // the binary at compile time. + buildArgs: { + type: 'object', + propertyNames: { type: 'string', pattern: '^[A-Za-z_][A-Za-z0-9_]*$' }, + additionalProperties: { type: 'string' }, + }, }, required: ['enabled', 'context', 'dockerFile', 'target'], additionalProperties: false, diff --git a/packages/dashmate/src/config/generateEnvsFactory.js b/packages/dashmate/src/config/generateEnvsFactory.js index 4992cd524f7..bc9f78c75bb 100644 --- a/packages/dashmate/src/config/generateEnvsFactory.js +++ b/packages/dashmate/src/config/generateEnvsFactory.js @@ -103,6 +103,40 @@ export default function generateEnvsFactory(configFile, homeDir, getConfigProfil ...convertObjectToEnvs(config.getOptions()), }; + // Forward extra docker `build.args` declared per-image in dashmate config + // (`platform.drive.abci.docker.build.buildArgs`, + // `platform.dapi.rsDapi.docker.build.buildArgs`) as env vars under the + // arg name. `docker-compose.build.*.yml` reads them via `${NAME}` + // substitution and forwards them into the Dockerfile `ARG`. + // + // Dashmate config is the single source of truth for build args — do NOT + // fall back to `process.env[key]`. Operators who need a per-invocation + // override should `yarn dashmate config set ...` rather than `FOO=bar + // yarn start`. Keeping this single-source matches the `yarn setup` flow + // that writes SDK_TEST_DATA into the local config automatically. + // + // drive-abci and rs-dapi share the workspace so a shared key like + // CARGO_BUILD_PROFILE typically wants the same value in both; the merge + // order below means drive-abci's value wins on collision. + const getBuildArgs = (configPath) => ( + config.has(configPath) ? (config.get(configPath) || {}) : {} + ); + const mergedBuildArgs = { + ...getBuildArgs('platform.dapi.rsDapi.docker.build.buildArgs'), + ...getBuildArgs('platform.drive.abci.docker.build.buildArgs'), + }; + const reservedEnvKeys = new Set([ + 'COMPOSE_FILE', 'COMPOSE_PROJECT_NAME', 'COMPOSE_PROFILES', 'COMPOSE_PATH_SEPARATOR', + 'DOCKER_BUILDKIT', 'COMPOSE_DOCKER_CLI_BUILD', 'CONFIG_NAME', 'DASHMATE_HOME_DIR', 'LOCAL_UID', 'LOCAL_GID', + ]); + for (const [key, value] of Object.entries(mergedBuildArgs)) { + // don't let buildArgs clobber reserved compose/runtime envs + if (reservedEnvKeys.has(key)) { + continue; + } + envs[key] = value; + } + const configuredAccessLogPath = config.get('platform.dapi.rsDapi.logs.accessLogPath'); const hasConfiguredPath = typeof configuredAccessLogPath === 'string' && configuredAccessLogPath.trim() !== ''; diff --git a/packages/dashmate/test/unit/config/Config.spec.js b/packages/dashmate/test/unit/config/Config.spec.js new file mode 100644 index 00000000000..0d10b89d6f9 --- /dev/null +++ b/packages/dashmate/test/unit/config/Config.spec.js @@ -0,0 +1,143 @@ +import { expect } from 'chai'; +import Config from '../../../src/config/Config.js'; + +describe('Config', () => { + describe('.isSchemaPathAllowed', () => { + // The bug that triggered this method: `dashmate config set + // platform.drive.abci.docker.build.buildArgs.SDK_TEST_DATA true` + // was failing because the old `config.get(path)` pre-check rejected + // any path whose value isn't already stored — including new keys + // inside map-shaped properties whose schema legally accepts them + // via `additionalProperties: `. Each test pins one slice + // of the schema-walk so a regression surfaces fast. + + describe('typed properties', () => { + it('accepts a deeply-nested path that traverses only `properties`', () => { + expect( + Config.isSchemaPathAllowed('platform.drive.abci.docker.build.enabled'), + ).to.be.true(); + }); + + it('accepts a top-level property', () => { + expect(Config.isSchemaPathAllowed('network')).to.be.true(); + }); + + it('rejects a top-level typo', () => { + // `platfom` (typo) is not in top-level `properties` and the schema + // has `additionalProperties: false`, so it must not be allowed. + expect( + Config.isSchemaPathAllowed('platfom.drive.abci.docker.build.enabled'), + ).to.be.false(); + }); + + it('rejects a typo in the middle of the path', () => { + expect( + Config.isSchemaPathAllowed('platform.drive.abc.docker.build.enabled'), + ).to.be.false(); + }); + }); + + describe('map-shaped properties (additionalProperties: )', () => { + it('accepts a new key inside a value-keyed map', () => { + // The original failure. `buildArgs` is defined as + // `{ type: 'object', additionalProperties: { type: 'string' } }` + // so any key with a string value is schema-legal even if not yet set. + expect( + Config.isSchemaPathAllowed( + 'platform.drive.abci.docker.build.buildArgs.SDK_TEST_DATA', + ), + ).to.be.true(); + }); + + it('accepts an arbitrary key inside the map (the whole point)', () => { + expect( + Config.isSchemaPathAllowed( + 'platform.drive.abci.docker.build.buildArgs.ANY_KEY_AT_ALL', + ), + ).to.be.true(); + }); + + it('accepts the map property itself', () => { + expect( + Config.isSchemaPathAllowed('platform.drive.abci.docker.build.buildArgs'), + ).to.be.true(); + }); + }); + + describe('$ref traversal', () => { + it('descends through a $ref to `#/definitions/dockerBuild`', () => { + // `platform.drive.abci.docker.build` resolves via $ref to the shared + // `dockerBuild` definition — buildArgs is defined there. + expect( + Config.isSchemaPathAllowed( + 'platform.drive.abci.docker.build.buildArgs.X', + ), + ).to.be.true(); + }); + + it('descends through a $ref for a sibling Rust build (rs-dapi)', () => { + expect( + Config.isSchemaPathAllowed( + 'platform.dapi.rsDapi.docker.build.buildArgs.X', + ), + ).to.be.true(); + }); + }); + + describe('edge cases', () => { + it('rejects an empty path', () => { + expect(Config.isSchemaPathAllowed('')).to.be.false(); + }); + + it('rejects a non-string path', () => { + expect(Config.isSchemaPathAllowed(null)).to.be.false(); + expect(Config.isSchemaPathAllowed(undefined)).to.be.false(); + expect(Config.isSchemaPathAllowed(42)).to.be.false(); + }); + + it('rejects descending past a leaf primitive', () => { + // `network` is a string at top level; you cannot index further. + expect(Config.isSchemaPathAllowed('network.something')).to.be.false(); + }); + }); + }); + + describe('regression: paths the buggy pre-check rejected are now permitted', () => { + // Before the fix, `dashmate config set` did a value-existence pre-check + // (`config.get(path)`) that threw `InvalidOptionPathError` for any path + // whose value wasn't already stored. That blocked legal sets under + // map-shaped properties (`additionalProperties: `). The fix + // replaced the pre-check with `isSchemaPathAllowed`. These tests pin + // each path the original failure surfaced through. + + it('permits the original failing path (`…buildArgs.SDK_TEST_DATA`)', () => { + expect( + Config.isSchemaPathAllowed( + 'platform.drive.abci.docker.build.buildArgs.SDK_TEST_DATA', + ), + ).to.be.true(); + }); + + it('permits the same shape for rs-dapi (parallel Rust build)', () => { + expect( + Config.isSchemaPathAllowed( + 'platform.dapi.rsDapi.docker.build.buildArgs.CARGO_BUILD_PROFILE', + ), + ).to.be.true(); + }); + + it('still permits the canonical typed paths that the pre-check used to handle', () => { + // Sanity: paths whose values DO exist after `dashmate setup local` — + // the original pre-check used to gate these via `config.get`. The + // schema walker must accept them too, or the CLI breaks for everyone. + for (const path of [ + 'platform.drive.abci.docker.build.enabled', + 'platform.drive.abci.docker.image', + 'platform.dapi.rsDapi.docker.build.enabled', + 'core.insight.enabled', + ]) { + expect(Config.isSchemaPathAllowed(path), `path: ${path}`).to.be.true(); + } + }); + }); +}); diff --git a/packages/rs-dpp/Cargo.toml b/packages/rs-dpp/Cargo.toml index ebfd571587c..8bcc4360d75 100644 --- a/packages/rs-dpp/Cargo.toml +++ b/packages/rs-dpp/Cargo.toml @@ -71,7 +71,7 @@ strum = { version = "0.26", features = ["derive"] } json-schema-compatibility-validator = { path = '../rs-json-schema-compatibility-validator', optional = true } once_cell = "1.19.0" tracing = { version = "0.1.41" } -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9", optional = true } +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "5eb7a5380a6e974513343352acfd6b30a8c1f87c", optional = true } [dev-dependencies] tokio = { version = "1.40", features = ["full"] } diff --git a/packages/rs-drive-abci/Cargo.toml b/packages/rs-drive-abci/Cargo.toml index ea3211d45a0..633d3d2db7e 100644 --- a/packages/rs-drive-abci/Cargo.toml +++ b/packages/rs-drive-abci/Cargo.toml @@ -82,8 +82,12 @@ derive_more = { version = "1.0", features = ["from", "deref", "deref_mut"] } async-trait = "0.1.77" console-subscriber = { version = "0.4", optional = true } bls-signatures = { git = "https://github.com/dashpay/bls-signatures", rev = "0842b17583888e8f46c252a4ee84cdfd58e0546f", optional = true } -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9" } +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "5eb7a5380a6e974513343352acfd6b30a8c1f87c" } nonempty = "0.11" +# Shielded-pool snapshot needs raw RocksDB SstFileWriter + ingest_external_file_cf +# bindings, and blake3 for the snapshot-file checksum. +rocksdb = { git = "https://github.com/QuantumExplorer/rust-rocksdb.git", rev = "52772eea7bcd214d1d07d80aa538b1d24e5015b7" } +blake3 = "1.5" [dev-dependencies] platform-version = { path = "../rs-platform-version", features = [ @@ -103,7 +107,7 @@ dpp = { path = "../rs-dpp", default-features = false, features = [ drive = { path = "../rs-drive", features = ["fixtures-and-mocks"] } drive-proof-verifier = { path = "../rs-drive-proof-verifier" } strategy-tests = { path = "../strategy-tests" } -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9", features = ["client"] } +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "5eb7a5380a6e974513343352acfd6b30a8c1f87c", features = ["client"] } assert_matches = "1.5.0" drive-abci = { path = ".", features = ["testing-config", "mocks"] } bls-signatures = { git = "https://github.com/dashpay/bls-signatures", rev = "0842b17583888e8f46c252a4ee84cdfd58e0546f" } @@ -115,6 +119,11 @@ libc = "0.2" rocksdb = { git = "https://github.com/QuantumExplorer/rust-rocksdb.git", rev = "52772eea7bcd214d1d07d80aa538b1d24e5015b7" } integer-encoding = { version = "4.0.0" } +# For dump_only_default_and_aux_cfs_under_shielded_subtree_prefix — same +# subtree-prefix algorithm grovedb uses internally. +grovedb-path = { git = "https://github.com/dashpay/grovedb", rev = "5eb7a5380a6e974513343352acfd6b30a8c1f87c" } +grovedb-storage = { git = "https://github.com/dashpay/grovedb", rev = "5eb7a5380a6e974513343352acfd6b30a8c1f87c" } + [features] default = ["bls-signatures"] mocks = ["mockall", "drive/fixtures-and-mocks", "bls-signatures"] diff --git a/packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/mod.rs b/packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/mod.rs index bd5494cab3a..8fd3d5bc133 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/mod.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/mod.rs @@ -7,6 +7,7 @@ use dpp::version::PlatformVersion; use drive::grovedb::TransactionArg; mod addresses; +mod shielded; mod tokens; impl Platform { @@ -21,15 +22,24 @@ impl Platform { transaction: TransactionArg, platform_version: &PlatformVersion, ) -> Result<(), Error> { - if self.config.network != Network::Regtest { - return Err(Error::Execution(ExecutionError::CorruptedCodeExecution( - "create_sdk_test_data must be called only on local network", - ))); + // Permit only non-production networks. Mainnet/Testnet must never + // carry SDK test fixtures (random identities, seeded shielded pool, + // pre-baked snapshot anchor that isn't a valid spend anchor). + // Regtest = local dashmate; Devnet = developer test networks + // (issue #3714 stress-test target). + match self.config.network { + Network::Regtest | Network::Devnet => {} + _ => { + return Err(Error::Execution(ExecutionError::CorruptedCodeExecution( + "create_sdk_test_data must be called only on local or devnet networks", + ))); + } } self.create_data_for_group_token_queries(block_info, transaction, platform_version)?; self.create_data_for_token_direct_prices(block_info, transaction, platform_version)?; self.create_data_for_addresses(block_info, transaction, platform_version)?; + self.create_data_for_shielded_pool(block_info, transaction, platform_version)?; Ok(()) } diff --git a/packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/shielded.rs b/packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/shielded.rs new file mode 100644 index 00000000000..8f2949e4f8d --- /dev/null +++ b/packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/shielded.rs @@ -0,0 +1,1137 @@ +//! Deterministic filler-note generator for the SDK genesis test-data seeder. +//! +//! Produces N filler notes — random valid Pallas-base `cmx` + random 32-byte +//! ρ + 216 random bytes of "ciphertext". No note is decryptable by any wallet +//! (ρ is not constrained to a valid `Nullifier`, so the SDK's compact +//! decryption short-circuits on the ρ-field check). For wallet-balance tests, +//! use real shielded-fund transitions post-genesis instead of seeded notes. +//! +//! All randomness comes from a single seeded `StdRng`. The GroveDB root hash +//! is byte-identical across hosts for a fixed `rng_seed`. + +use dpp::block::block_info::BlockInfo; +use dpp::version::PlatformVersion; +use drive::grovedb::Element; +use drive::grovedb::TransactionArg; +use drive::grovedb_path::SubtreePath; +use drive::grovedb_storage::{Storage, StorageBatch}; +use grovedb_commitment_tree::{merkle_hash_from_bytes, CommitmentTree, DashMemo}; +use rand::rngs::StdRng; +use rand::{RngCore, SeedableRng}; + +use crate::error::execution::ExecutionError; +use crate::error::Error; +use crate::platform_types::platform::Platform; + +/// Block height at which we record the genesis post-seed anchor. Matches +/// production's first end-of-block anchor (`run_block_proposal` at the end of +/// block 1), so the seeded chain is root-identical to a production chain at +/// height 1 (modulo seeded note contents). See design doc §5.4. +const GENESIS_ANCHOR_HEIGHT: u64 = 1; + +/// Dash Orchard wire size: 32 (epk) + 104 (enc_ciphertext, DashMemo) + 80 (out_ciphertext). +/// +/// Pinned against the SDK's parser in +/// `packages/rs-sdk/src/platform/shielded/notes_sync/decrypt.rs`. If that file +/// changes its expected layout, the unit test below catches it. +pub const ENCRYPTED_NOTE_WIRE_LEN: usize = 32 + 104 + 80; +const _: () = assert!(ENCRYPTED_NOTE_WIRE_LEN == 216); + +/// Configuration for the seeder. +/// +/// The chain's `create_data_for_shielded_pool` uses [`Self::sdk_test_data`] +/// (hardcoded const), so every SDK_TEST_DATA devnet seeds the same N-note +/// pool regardless of operator env. Tests construct custom configs directly +/// to vary N. +#[derive(Debug, Clone)] +pub struct ShieldedSeedConfig { + /// Total filler notes to seed. + pub total_notes: u32, + /// RNG seed; identical seed ⇒ identical root hash. + pub rng_seed: u64, +} + +impl Default for ShieldedSeedConfig { + fn default() -> Self { + Self { + total_notes: 0, + rng_seed: 0xDEAD_BEEF, + } + } +} + +impl ShieldedSeedConfig { + /// The hardcoded SDK_TEST_DATA seed config used at every devnet genesis. + /// + /// `total_notes = 1_000_000`, seed `0xDEAD_BEEF` is fixed so the GroveDB + /// root hash is byte-identical across hosts. + /// + /// Wallet-balance UX no longer comes from seeded notes (the test wallet + /// hardcoding was removed). For decryptable balance flows in tests, use + /// real shielded-fund transitions post-genesis instead. + /// + /// At 1M notes the bake step takes ~5-15 min in the docker buildkit + /// linux VM (release profile required — see + /// `scripts/setup_local_network.sh::CARGO_BUILD_PROFILE`). Apply at + /// runtime InitChain remains ~134 ms because it's a single SST ingest + /// + parent-Merk leaf update regardless of N. + pub const fn sdk_test_data() -> Self { + Self { + total_notes: 1_000_000, + rng_seed: 0xDEAD_BEEF, + } + } +} + +/// A single seeded filler note in the BulkAppendTree input shape. +#[derive(Debug, Clone)] +pub struct SeededNote { + pub cmx: [u8; 32], + /// On-wire `nullifier` field — this is ρ, *not* the spend-time revealed + /// nullifier. + pub rho: [u8; 32], + /// 216 bytes: `epk(32) || enc_ciphertext(104) || out_ciphertext(80)`. + pub encrypted_note: Vec, +} + +/// Rejection-sample a valid Pallas base field element from the seeded RNG. +fn sample_valid_pallas_base(rng: &mut StdRng) -> [u8; 32] { + loop { + let mut bytes = [0u8; 32]; + rng.fill_bytes(&mut bytes); + if merkle_hash_from_bytes(&bytes).is_some() { + return bytes; + } + } +} + +/// Build one filler note. ρ is intentionally random 32 bytes (not necessarily +/// a valid Pallas element) so the SDK's compact decryption short-circuits on +/// the field check — guarantees the filler is not decryptable by any wallet. +fn generate_filler_note(rng: &mut StdRng) -> SeededNote { + let cmx = sample_valid_pallas_base(rng); + let mut rho = [0u8; 32]; + rng.fill_bytes(&mut rho); + let mut encrypted_note = vec![0u8; ENCRYPTED_NOTE_WIRE_LEN]; + rng.fill_bytes(&mut encrypted_note); + SeededNote { + cmx, + rho, + encrypted_note, + } +} + +/// Generate `count` filler notes from the seeded `rng`. Each call advances +/// `rng` deterministically, so chained calls across batches still produce +/// the byte-identical sequence that a single up-front call would have made. +pub fn generate_filler_batch(rng: &mut StdRng, count: usize) -> Vec { + (0..count).map(|_| generate_filler_note(rng)).collect() +} + +/// Generate every seeded filler note up front. Equivalent to one +/// [`generate_filler_batch`] call sized to `cfg.total_notes`; only used by +/// tests that prefer the eager shape — the production seed path streams +/// batches via `generate_filler_batch` directly. +#[cfg(test)] +fn generate_notes(cfg: &ShieldedSeedConfig) -> Vec { + let mut rng = StdRng::seed_from_u64(cfg.rng_seed); + generate_filler_batch(&mut rng, cfg.total_notes as usize) +} + +impl Platform { + /// Env-based entrypoint called by `create_sdk_test_data`. Reads + /// `SHIELDED_SEED_*` and delegates to [`Self::seed_shielded_pool_with_config`]. + /// + /// When `SHIELDED_SEED_TOTAL_NOTES = 0` (the default), no notes are seeded + /// but the anchor recorder still runs — matching production's + /// end-of-block-1 behaviour on an empty pool. + /// + /// **`None` transaction is tolerated**: production (`init_chain`) always + /// passes a transaction, but several test helpers (notably + /// `TestPlatformBuilder::set_genesis_state`) invoke `create_genesis_state` + /// with `None`. In that case we skip both the seeding and the anchor + /// recording — those tests aren't exercising the shielded pool, and the + /// next test step that does will pass a tx through + /// `seed_shielded_pool_with_config` explicitly. + pub(super) fn create_data_for_shielded_pool( + &self, + block_info: &BlockInfo, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + if transaction.is_none() { + tracing::debug!( + "create_data_for_shielded_pool: no transaction; skipping seeding \ + (test path — production always supplies a tx)" + ); + return Ok(()); + } + + // Fast-path: if `DRIVE_SHIELDED_SNAPSHOT` is set, load the + // precomputed snapshot via SST ingest instead of running the (very + // slow) runtime seeder. The snapshot file is produced offline by the + // snapshot-bake binary against a chain whose `ShieldedSeedConfig` + // matches what this build expects. + // + // Any failure (file missing, magic mismatch, version skew, checksum + // failure, cross-validation drift) is FATAL — we surface the error + // and InitChain fails loudly. No silent fallback to the seeder; the + // operator is expected to fix the snapshot, not silently get a chain + // with different state than they asked for. + if let Ok(snapshot_path) = std::env::var("DRIVE_SHIELDED_SNAPSHOT") { + let path = std::path::PathBuf::from(&snapshot_path); + tracing::info!( + snapshot_path = %path.display(), + "create_data_for_shielded_pool: applying precomputed shielded snapshot" + ); + let stats = crate::shielded_snapshot::apply_shielded_snapshot( + &self.drive.grove, + transaction, + &path, + platform_version, + ) + .map_err(|e| { + Error::Execution(ExecutionError::Conversion(format!( + "DRIVE_SHIELDED_SNAPSHOT apply failed: {e}" + ))) + })?; + tracing::info!( + total_count = stats.total_count, + combined_root = format!("{}", hex::encode(stats.combined_root)), + "create_data_for_shielded_pool: snapshot applied" + ); + // Record the anchor at height 1 so wallets see the same anchor + // they would after a runtime seed. block_info is unused here + // (record_shielded_pool_anchor_if_changed takes block_height + // directly, and the bake step happens at genesis). + let _ = block_info; + let tx = + transaction.ok_or(Error::Execution(ExecutionError::CorruptedCodeExecution( + "create_data_for_shielded_pool snapshot path requires a transaction", + )))?; + self.drive + .record_shielded_pool_anchor_if_changed(GENESIS_ANCHOR_HEIGHT, tx, platform_version) + .map_err(Into::::into) + .map(|_| ())?; + return Ok(()); + } + + let cfg = ShieldedSeedConfig::sdk_test_data(); + tracing::info!( + total_notes = cfg.total_notes, + rng_seed = format!("0x{:x}", cfg.rng_seed), + "create_data_for_shielded_pool: seeding SDK_TEST_DATA shielded pool" + ); + self.seed_shielded_pool_with_config(&cfg, block_info, transaction, platform_version) + } + + /// Seed the shielded pool with deterministic test data using the supplied + /// config, then record the post-seed anchor at height 1. + /// + /// All randomness comes from `cfg.rng_seed`; identical config ⇒ identical + /// GroveDB root hash. See design doc §5.3 + §5.4. Exposed for integration + /// tests so they can pin specific configs without depending on env-var + /// state. + pub(super) fn seed_shielded_pool_with_config( + &self, + cfg: &ShieldedSeedConfig, + block_info: &BlockInfo, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + tracing::info!( + cfg_total_notes = cfg.total_notes, + cfg_rng_seed = format!("0x{:x}", cfg.rng_seed), + "seed_shielded_pool_with_config: entered" + ); + if cfg.total_notes > 0 { + tracing::info!( + total_notes = cfg.total_notes, + rng_seed = format!("0x{:x}", cfg.rng_seed), + "seeding shielded pool with SDK test data (batched append_many_raw)" + ); + let tx = + transaction.ok_or(Error::Execution(ExecutionError::CorruptedCodeExecution( + "seed_shielded_pool_with_config requires a transaction", + )))?; + + // Read parent-Merk Element to get chunk_power + flags. The batched + // seed path requires `total_count == 0` at entry — the running + // `CommitmentTree` is opened against an empty subtree and we + // accumulate `total_notes` appends on top. + let pool_path_arr = drive::drive::shielded::paths::shielded_credit_pool_path(); + let leaf_key = &[drive::drive::shielded::paths::SHIELDED_NOTES_KEY]; + let parent_path = SubtreePath::from(pool_path_arr.as_slice()); + let element = self + .drive + .grove + .get_raw( + parent_path.clone(), + leaf_key, + Some(tx), + &platform_version.drive.grove_version, + ) + .value + .map_err(|e| { + Error::Execution(ExecutionError::CorruptedCodeExecution(Box::leak( + format!("seed: get_raw parent leaf: {e}").into_boxed_str(), + ))) + })?; + let (initial_total_count, chunk_power, flags) = match element { + Element::CommitmentTree(tc, cp, f) => (tc, cp, f), + _ => { + return Err(Error::Execution(ExecutionError::CorruptedCodeExecution( + "seed: parent leaf is not Element::CommitmentTree", + ))) + } + }; + assert_eq!( + initial_total_count, 0, + "batched seed requires an empty commitment tree" + ); + + // Open a CommitmentTree on the subtree's storage context. The + // StorageBatch buffers every BulkAppendTree / frontier write and + // is committed once at the end via `commit_multi_context_batch` + // — same shape as the production `commitment_tree_insert` path. + let subtree_path_segs: Vec> = pool_path_arr + .iter() + .map(|s| s.to_vec()) + .chain(std::iter::once(leaf_key.to_vec())) + .collect(); + let subtree_path_refs: Vec<&[u8]> = + subtree_path_segs.iter().map(|v| v.as_slice()).collect(); + let subtree_path = SubtreePath::from(subtree_path_refs.as_slice()); + + let data_batch = StorageBatch::new(); + let storage_ctx = self + .drive + .grove + .raw_storage() + .get_transactional_storage_context(subtree_path, Some(&data_batch), tx) + .unwrap(); + let mut ct = CommitmentTree::<_, DashMemo>::open(0, chunk_power, storage_ctx) + .value + .map_err(|e| { + Error::Execution(ExecutionError::CorruptedCodeExecution(Box::leak( + format!("seed: CommitmentTree::open: {e}").into_boxed_str(), + ))) + })?; + + // Batched seed via repeated `append_many_raw` (grovedb PR #751). + // Each batch: + // 1. Generates `BATCH_SIZE` filler notes from the seeded RNG. + // 2. Calls `append_many_raw` — byte-for-byte equivalent to N × + // `append_raw` per the upstream contract; flushes the MMR + // overlay internally. + // 3. `ct.save()` persists the Sinsemilla frontier into the + // `data_batch` (still in-memory until the final commit). + // 4. Logs the running anchor + bulk-state root so progress is + // observable and post-hoc replay is debuggable. + // + // The whole `data_batch` is committed once at the end. A crash + // mid-bake loses every batch; durability per batch would require + // a per-batch `commit_multi_context_batch`. For a bake binary + // that lives in a single Docker stage producing an artifact, the + // simpler shape is fine (re-run on failure). + const BATCH_SIZE: usize = 10_000; + let total = cfg.total_notes as usize; + let bake_start = std::time::Instant::now(); + tracing::info!( + total, + batch_size = BATCH_SIZE, + "seed: starting batched commitment-tree append" + ); + let mut rng = StdRng::seed_from_u64(cfg.rng_seed); + let mut appended_total = 0usize; + let mut last_sinsemilla_root: [u8; 32] = [0u8; 32]; + let mut last_bulk_state_root: [u8; 32] = [0u8; 32]; + let mut batch_index = 0usize; + while appended_total < total { + let this_batch = std::cmp::min(BATCH_SIZE, total - appended_total); + let batch_start = std::time::Instant::now(); + let batch_notes = generate_filler_batch(&mut rng, this_batch); + let gen_elapsed = batch_start.elapsed(); + + let iter = batch_notes + .into_iter() + .map(|n| (n.cmx, n.rho, n.encrypted_note)); + let append_result = ct.append_many_raw(iter).value.map_err(|e| { + Error::Execution(ExecutionError::CorruptedCodeExecution(Box::leak( + format!("seed: append_many_raw (batch {batch_index}): {e}") + .into_boxed_str(), + ))) + })?; + // Persist the Sinsemilla frontier per batch — cheap and + // gives durable mid-bake checkpoints if we ever want to + // resume from a crash. Mid-bake MMR `commit_mmr` is + // intentionally NOT called here: per the upstream fix + // `1340db71 fix(commitment-tree): don't commit_mmr inside + // append_many_raw`, the MMR's compaction state is + // accumulated in memory across `append_many_raw` calls and + // flushed exactly once at the end. Calling `commit_mmr` + // between batches corrupts the in-memory overlay (manifests + // as "MMR get_root failed: Inconsistent store" on the next + // call). One final `commit_mmr` follows the loop below. + ct.save().value.map_err(|e| { + Error::Execution(ExecutionError::CorruptedCodeExecution(Box::leak( + format!("seed: ct.save (batch {batch_index}): {e}").into_boxed_str(), + ))) + })?; + + last_sinsemilla_root = append_result.sinsemilla_root; + last_bulk_state_root = append_result.bulk_state_root; + appended_total += this_batch; + batch_index += 1; + + let total_elapsed = bake_start.elapsed(); + let rate = appended_total as f64 / total_elapsed.as_secs_f64().max(0.001); + let remaining = total.saturating_sub(appended_total); + let eta_secs = if rate > 0.0 { + (remaining as f64 / rate) as u64 + } else { + 0 + }; + tracing::info!( + batch_index, + batch_size = this_batch, + appended = appended_total, + total, + pct = format!( + "{:.1}%", + (appended_total as f64 / total as f64) * 100.0 + ), + elapsed_s = total_elapsed.as_secs(), + batch_elapsed_ms = batch_start.elapsed().as_millis() as u64, + gen_elapsed_ms = gen_elapsed.as_millis() as u64, + rate_per_s = format!("{:.0}", rate), + eta_s = eta_secs, + sinsemilla_root = %hex::encode(last_sinsemilla_root), + bulk_state_root = %hex::encode(last_bulk_state_root), + "seed batch complete" + ); + } + + // Single MMR flush after every batch has appended. The MMR + // overlay accumulates across `append_many_raw` calls and is + // persisted only here. See the in-loop comment above and the + // upstream fix `1340db71` for the rationale. + ct.commit_mmr().map_err(|e| { + Error::Execution(ExecutionError::CorruptedCodeExecution(Box::leak( + format!("seed: ct.commit_mmr (final): {e}").into_boxed_str(), + ))) + })?; + + let combined_root = grovedb_commitment_tree::compute_commitment_tree_state_root( + &last_sinsemilla_root, + &last_bulk_state_root, + ); + tracing::info!( + total, + batches = batch_index, + elapsed_s = bake_start.elapsed().as_secs(), + combined_root = %hex::encode(combined_root), + "seed: all batches complete" + ); + drop(ct); + + // Commit the subtree's data_batch through the transaction — + // makes all the BulkAppendTree / frontier writes visible to + // subsequent reads (including replace_commitment_tree_subtree_root's + // own batch). + self.drive + .grove + .raw_storage() + .commit_multi_context_batch(data_batch, Some(tx)) + .value + .map_err(|e| { + Error::Execution(ExecutionError::CorruptedCodeExecution(Box::leak( + format!("seed: commit_multi_context_batch: {e}").into_boxed_str(), + ))) + })?; + + // --- Update parent Merk leaf with the new state --- + self.drive + .grove + .replace_commitment_tree_subtree_root( + parent_path, + leaf_key, + u64::from(cfg.total_notes), + chunk_power, + flags, + combined_root, + Some(tx), + &platform_version.drive.grove_version, + ) + .value + .map_err(|e| { + Error::Execution(ExecutionError::CorruptedCodeExecution(Box::leak( + format!("seed: replace_commitment_tree_subtree_root: {e}").into_boxed_str(), + ))) + })?; + + // Post-bake assertion — catches silent truncation from a panic + // mid-bake (per design doc §15.6 F9). + let mut drive_ops = vec![]; + let count_after = + self.drive + .shielded_pool_notes_count(Some(tx), &mut drive_ops, platform_version)?; + assert_eq!( + count_after, + u64::from(cfg.total_notes), + "seed: post-bake count mismatch (expected {}, got {})", + cfg.total_notes, + count_after + ); + + // block_info is unused on this fast path (no per-note state + // transitions, no fee accounting) — accepted as a signature + // artifact for compatibility. + let _ = block_info; + } + + // Always record the post-seed anchor at height 1 — matches production's + // first end-of-block anchor. With cfg.total_notes == 0 this records the + // empty-tree Sinsemilla root, which is the same value production + // records at end of block 1 against an empty pool. See design doc §5.4 + // for why one anchor suffices (wallet creates a single checkpoint at + // post-sync tree size). + let tx = transaction.ok_or(Error::Execution(ExecutionError::CorruptedCodeExecution( + "create_data_for_shielded_pool requires a transaction", + )))?; + self.drive + .record_shielded_pool_anchor_if_changed(GENESIS_ANCHOR_HEIGHT, tx, platform_version) + .map_err(Error::Drive)?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashSet; + + fn small_cfg() -> ShieldedSeedConfig { + ShieldedSeedConfig { + total_notes: 16, + rng_seed: 0xDEAD_BEEF, + } + } + + #[test] + fn generate_notes_count_matches_total() { + let cfg = small_cfg(); + let notes = generate_notes(&cfg); + assert_eq!(notes.len(), cfg.total_notes as usize); + } + + #[test] + fn generate_notes_filler_ciphertext_size_pinned_to_216() { + let cfg = small_cfg(); + let notes = generate_notes(&cfg); + for n in ¬es { + assert_eq!( + n.encrypted_note.len(), + ENCRYPTED_NOTE_WIRE_LEN, + "all encrypted_note payloads must be 216 bytes — matches \ + packages/rs-sdk/src/platform/shielded/notes_sync/decrypt.rs" + ); + } + } + + #[test] + fn generate_notes_is_deterministic() { + let cfg = small_cfg(); + let a = generate_notes(&cfg); + let b = generate_notes(&cfg); + assert_eq!(a.len(), b.len()); + for (i, (na, nb)) in a.iter().zip(b.iter()).enumerate() { + assert_eq!(na.cmx, nb.cmx, "cmx differs at position {}", i); + assert_eq!(na.rho, nb.rho, "rho differs at position {}", i); + assert_eq!( + na.encrypted_note, nb.encrypted_note, + "encrypted_note differs at position {}", + i + ); + } + } + + #[test] + fn generate_notes_changes_with_different_seed() { + let cfg_a = ShieldedSeedConfig { + rng_seed: 1, + ..small_cfg() + }; + let cfg_b = ShieldedSeedConfig { + rng_seed: 2, + ..small_cfg() + }; + let a = generate_notes(&cfg_a); + let b = generate_notes(&cfg_b); + // At least one cmx differs (the first one — different RNG stream). + assert!(a.iter().zip(b.iter()).any(|(na, nb)| na.cmx != nb.cmx)); + } + + /// The batched generator must produce byte-identical output to a single + /// up-front call. Otherwise the seeded GroveDB state diverges between + /// "one big append_many_raw" and "N × append_many_raw(BATCH_SIZE)". + #[test] + fn batched_generator_matches_single_call() { + let cfg = ShieldedSeedConfig { + total_notes: 100, + rng_seed: 0xDEAD_BEEF, + }; + let one_shot = generate_notes(&cfg); + + let mut rng = StdRng::seed_from_u64(cfg.rng_seed); + let mut batched: Vec = Vec::with_capacity(100); + for _ in 0..10 { + batched.extend(generate_filler_batch(&mut rng, 10)); + } + assert_eq!(one_shot.len(), batched.len()); + for (i, (a, b)) in one_shot.iter().zip(batched.iter()).enumerate() { + assert_eq!(a.cmx, b.cmx, "cmx differs at {i}"); + assert_eq!(a.rho, b.rho, "rho differs at {i}"); + assert_eq!(a.encrypted_note, b.encrypted_note, "ct differs at {i}"); + } + } + + /// Sanity: 1k filler notes have unique cmx values with overwhelming + /// probability (Pallas base field is enormous, collisions are + /// cryptographically negligible). A regression where cmx is no longer + /// drawn freshly per call would surface as a duplicate here. + #[test] + fn generated_cmx_values_are_unique() { + let cfg = ShieldedSeedConfig { + total_notes: 1024, + rng_seed: 0xDEAD_BEEF, + }; + let notes = generate_notes(&cfg); + let mut seen: HashSet<[u8; 32]> = HashSet::with_capacity(notes.len()); + for n in ¬es { + assert!(seen.insert(n.cmx), "duplicate cmx in generator output"); + } + } +} + +#[cfg(test)] +mod platform_tests { + //! End-to-end tests that drive `seed_shielded_pool_with_config` against a + //! real `Platform` instance. The pure-function tests for the note generator + //! live in the sibling `tests` module above; these tests verify the + //! integration with Drive (count, anchor, determinism) — i.e. that + //! production's `apply_drive_operations` ⇒ `commitment_tree_insert_op` path + //! ends up consistent with what the generator produced. + + use super::*; + use crate::config::PlatformConfig; + use crate::test::helpers::setup::TestPlatformBuilder; + use drive::drive::shielded::paths::{shielded_credit_pool_path, SHIELDED_NOTES_KEY}; + use grovedb_commitment_tree::EMPTY_SINSEMILLA_ROOT; + + /// Reduced default for integration tests — smaller is faster and still + /// exercises the batched seed path on small N. + fn integration_cfg() -> ShieldedSeedConfig { + ShieldedSeedConfig { + total_notes: 16, + rng_seed: 0xDEAD_BEEF, + } + } + + /// `set_genesis_state` runs `create_sdk_test_data` under the cfg flag, which + /// errors unless the platform is on the `Regtest` network. The default + /// `TestPlatformBuilder::new()` config is mainnet, so every test in this + /// module has to switch to regtest before calling `set_genesis_state`. + fn build_regtest_platform( + ) -> crate::test::helpers::setup::TempPlatform { + TestPlatformBuilder::new() + .with_config(PlatformConfig::default_local()) + .build_with_mock_rpc() + .set_genesis_state() + } + + /// Read the current Sinsemilla anchor for the credit shielded pool. + fn read_current_anchor( + platform: &Platform, + transaction: TransactionArg, + platform_version: &PlatformVersion, + ) -> [u8; 32] { + let pool_path = shielded_credit_pool_path(); + platform + .drive + .grove + .commitment_tree_anchor( + &pool_path, + &[SHIELDED_NOTES_KEY], + transaction, + &platform_version.drive.grove_version, + ) + .unwrap() + .expect("commitment_tree_anchor") + .to_bytes() + } + + /// Build a fresh test platform, seed with `cfg`, return the post-seed Sinsemilla + /// anchor read inside the same transaction. + fn build_and_seed(cfg: &ShieldedSeedConfig, platform_version: &PlatformVersion) -> [u8; 32] { + let platform = build_regtest_platform(); + let tx = platform.drive.grove.start_transaction(); + platform + .seed_shielded_pool_with_config(cfg, &BlockInfo::default(), Some(&tx), platform_version) + .expect("seed must succeed"); + read_current_anchor(&platform.platform, Some(&tx), platform_version) + } + + #[test] + fn empty_config_leaves_pool_at_empty_sinsemilla_root() { + // After `set_genesis_state`, the env-based call ran with total_notes=0 + // (no SHIELDED_SEED_* envs set in this test), so the live Sinsemilla + // frontier must equal the well-known empty-tree root. + let platform_version = PlatformVersion::latest(); + let platform = build_regtest_platform(); + let tx = platform.drive.grove.start_transaction(); + let anchor = read_current_anchor(&platform, Some(&tx), platform_version); + assert_eq!(anchor, EMPTY_SINSEMILLA_ROOT); + } + + #[test] + fn seeded_pool_count_matches_total_notes() { + let platform_version = PlatformVersion::latest(); + let platform = build_regtest_platform(); + let tx = platform.drive.grove.start_transaction(); + let cfg = integration_cfg(); + platform + .seed_shielded_pool_with_config( + &cfg, + &BlockInfo::default(), + Some(&tx), + platform_version, + ) + .expect("seed"); + + let mut drive_ops = vec![]; + let count = platform + .drive + .shielded_pool_notes_count(Some(&tx), &mut drive_ops, platform_version) + .expect("shielded_pool_notes_count"); + assert_eq!(count, u64::from(cfg.total_notes)); + } + + #[test] + fn seeded_anchor_differs_from_empty_root() { + let platform_version = PlatformVersion::latest(); + let cfg = integration_cfg(); + let anchor = build_and_seed(&cfg, platform_version); + assert_ne!( + anchor, EMPTY_SINSEMILLA_ROOT, + "post-seed anchor must differ from the empty-tree root" + ); + } + + #[test] + fn seeding_with_same_config_is_byte_identical_across_platforms() { + // The load-bearing determinism test: two fresh platforms, same config, + // must produce byte-identical Sinsemilla anchors. If this ever fails, + // some part of the generator is consuming `OsRng` / `thread_rng()` + // instead of the seeded RNG, or a transitive dep changed Pallas-field + // semantics under us. + let platform_version = PlatformVersion::latest(); + let cfg = integration_cfg(); + let anchor1 = build_and_seed(&cfg, platform_version); + let anchor2 = build_and_seed(&cfg, platform_version); + assert_eq!( + anchor1, anchor2, + "same config must produce byte-identical anchors across runs" + ); + } + + #[test] + fn different_rng_seeds_produce_different_anchors() { + let platform_version = PlatformVersion::latest(); + let cfg_1 = ShieldedSeedConfig { + rng_seed: 1, + ..integration_cfg() + }; + let cfg_2 = ShieldedSeedConfig { + rng_seed: 2, + ..integration_cfg() + }; + let a1 = build_and_seed(&cfg_1, platform_version); + let a2 = build_and_seed(&cfg_2, platform_version); + assert_ne!(a1, a2); + } + + #[test] + fn seeding_zero_notes_keeps_pool_at_empty_root() { + // Explicit zero-notes config must produce the empty-tree root, matching + // the no-config path. Pins the §5.4 claim that N=0 is a no-op for the + // commitment tree but still triggers the anchor recorder. + let platform_version = PlatformVersion::latest(); + let cfg = ShieldedSeedConfig { + total_notes: 0, + ..ShieldedSeedConfig::default() + }; + let anchor = build_and_seed(&cfg, platform_version); + assert_eq!(anchor, EMPTY_SINSEMILLA_ROOT); + } + + /// Native bake-feasibility benchmark — runs the FULL seeder (default N=500k, + /// overridable via `BENCH_N`) outside any Docker file-share layer and prints + /// wall-clock to stderr. The grovedb tempdir lands in `$TMPDIR`, which on + /// macOS is APFS (fast native fsync) and on linux CI is typically tmpfs. + /// + /// Compare-against: + /// - macOS Docker Desktop file-share, N=500k, release: ~3h 41m (measured). + /// - Same host natively (this test), N=500k, release: . + /// - Linux buildkit VM (where the bake stage would run), N=500k, release: + /// should be at most this number, usually faster. + /// + /// Hard-fail if the seed doesn't complete in 30 min — that's the threshold + /// past which the bake approach itself is in trouble. + /// + /// Run with: + /// ```bash + /// cargo test -p drive-abci --release --features create_sdk_test_data \ + /// bench_native_seed_full -- --ignored --nocapture + /// ``` + #[test] + #[ignore = "long-running bake-feasibility benchmark, see fn docs"] + fn bench_native_seed_full() { + let n: u32 = std::env::var("BENCH_N") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(500_000); + + let platform_version = PlatformVersion::latest(); + let platform = build_regtest_platform(); + + let cfg = ShieldedSeedConfig { + total_notes: n, + rng_seed: 0xDEAD_BEEF, + }; + + let tempdir_path = platform.tempdir.path().display().to_string(); + eprintln!( + "bench_native_seed_full: N={} cfg={:?} tempdir={}", + n, cfg, tempdir_path, + ); + + let tx = platform.drive.grove.start_transaction(); + let start = std::time::Instant::now(); + platform + .seed_shielded_pool_with_config( + &cfg, + &BlockInfo::default(), + Some(&tx), + platform_version, + ) + .expect("seed_shielded_pool_with_config must succeed"); + let seed_elapsed = start.elapsed(); + + let commit_start = std::time::Instant::now(); + tx.commit().expect("commit"); + let commit_elapsed = commit_start.elapsed(); + + let total = seed_elapsed + commit_elapsed; + eprintln!( + "bench_native_seed_full: N={} seed={:.2?} commit={:.2?} total={:.2?} \ + ({:.1} notes/sec)", + n, + seed_elapsed, + commit_elapsed, + total, + n as f64 / total.as_secs_f64(), + ); + + // Anchor sanity check so we know the work was real, not a noop. + let anchor = read_current_anchor(&platform, None, platform_version); + assert_ne!( + anchor, EMPTY_SINSEMILLA_ROOT, + "post-seed anchor must differ from empty (proves seeding actually ran)", + ); + + assert!( + total.as_secs() < 30 * 60, + "seed took {:?} for N={}; bake approach assumes native seeding is well \ + under 30 min — investigate if this fails", + total, + n, + ); + } + + /// Empirically verify the snapshot design's data-location claim — that the + /// shielded commitment-tree subtree's RocksDB state lives entirely in the + /// `default` + `aux` CFs under a deterministic subtree prefix. If + /// `roots` or `meta` CFs ALSO have entries for our subtree prefix, the + /// snapshot file format needs to include them and `apply_shielded_snapshot` + /// must ingest into 4 CFs not 2. + /// + /// What this proves: + /// 1. The subtree prefix is path-deterministic (built via + /// `RocksDbStorage::build_prefix` from `[shielded_credit_pool_path().., + /// SHIELDED_NOTES_KEY]`). + /// 2. After seeding N notes, the CFs with non-zero key counts under that + /// prefix tell us which CFs the dump must include. + /// 3. Counts scale with N as expected (default CF has chunks + buffer + MMR + /// + META; aux CF has exactly 1 entry, the Sinsemilla frontier). + #[test] + #[ignore = "raw-rocksdb introspection; long-ish setup"] + fn dump_only_default_and_aux_cfs_under_shielded_subtree_prefix() { + use grovedb_path::SubtreePath; + use grovedb_storage::rocksdb_storage::RocksDbStorage; + use rocksdb::{ColumnFamilyDescriptor, OptimisticTransactionDB, Options}; + + let platform_version = PlatformVersion::latest(); + let temp = build_regtest_platform(); + + // Seed a small N so the test is fast but still exercises every key + // family the BulkAppendTree writes (META, ≥1 buffer entry, ≥1 chunk if + // N ≥ 2^chunk_power). + let cfg = ShieldedSeedConfig { + total_notes: 4096, // > one chunk at chunk_power=11 → exercises e/m keys + rng_seed: 0xDEAD_BEEF, + }; + let tx = temp.drive.grove.start_transaction(); + temp.seed_shielded_pool_with_config( + &cfg, + &BlockInfo::default(), + Some(&tx), + platform_version, + ) + .expect("seed"); + tx.commit().expect("commit"); + let anchor = read_current_anchor(&temp, None, platform_version); + assert_ne!(anchor, EMPTY_SINSEMILLA_ROOT); + + // Compute the shielded subtree's deterministic prefix using grovedb's + // own helper — same algorithm production uses, no reimplementation. + let pool = drive::drive::shielded::paths::shielded_credit_pool_path_vec(); + let mut path_segments: Vec> = pool; + path_segments.push(vec![drive::drive::shielded::paths::SHIELDED_NOTES_KEY]); + let path = SubtreePath::from(path_segments.as_slice()); + let prefix = RocksDbStorage::build_prefix(path).unwrap(); + let prefix_bytes: [u8; 32] = prefix.into(); + eprintln!("subtree_prefix = {}", hex::encode(prefix_bytes)); + + // Close the Platform so we can take an exclusive lock on RocksDB. + // Destructure to keep the TempDir alive while the Platform's GroveDb + // handle is dropped — otherwise dropping the whole TempPlatform also + // drops the TempDir and deletes the directory underneath us. + let crate::test::helpers::setup::TempPlatform { + platform: pf, + tempdir, + } = temp; + let tempdir_path = tempdir.path().to_path_buf(); + drop(pf); + + // Open the same DB directly via rocksdb. CF names match grovedb's + // pub(crate) consts in `grovedb-storage/src/rocksdb_storage/storage.rs`: + // "default" is the implicit primary CF, the other three are named. + // If those names ever change in grovedb, this test must be updated; + // the production snapshot code will need the same update. + let cf_names = ["default", "aux", "roots", "meta"]; + let opts = { + let mut o = Options::default(); + o.create_if_missing(false); + o.create_missing_column_families(false); + o + }; + let cf_descs: Vec = cf_names + .iter() + .map(|n| ColumnFamilyDescriptor::new(*n, Options::default())) + .collect(); + let db: OptimisticTransactionDB = + OptimisticTransactionDB::open_cf_descriptors(&opts, &tempdir_path, cf_descs) + .expect("open A rocksdb"); + + // For each CF, count keys whose first 32 bytes equal the subtree prefix. + // Capture the first few key bytes too so a regression has actionable + // info, not just a count mismatch. + let mut per_cf_counts: Vec<(&str, usize, Vec>)> = vec![]; + for cf_name in cf_names { + let cf = db.cf_handle(cf_name).unwrap_or_else(|| { + panic!("missing CF {cf_name} — grovedb's CF layout may have changed") + }); + let mut iter = db.raw_iterator_cf(&cf); + iter.seek(&prefix_bytes[..]); + let mut count = 0usize; + let mut samples: Vec> = vec![]; + while iter.valid() { + let k = match iter.key() { + Some(k) => k, + None => break, + }; + if !k.starts_with(&prefix_bytes) { + break; + } + count += 1; + if samples.len() < 5 { + samples.push(k.to_vec()); + } + iter.next(); + } + eprintln!( + "CF {:>7}: {} keys under subtree prefix (first {} sampled)", + cf_name, + count, + samples.len(), + ); + for s in &samples { + eprintln!(" sample key (len={}): {}", s.len(), hex::encode(s)); + } + per_cf_counts.push((cf_name, count, samples)); + } + drop(db); + + let counts: std::collections::HashMap<&str, usize> = + per_cf_counts.iter().map(|(n, c, _)| (*n, *c)).collect(); + + let default_count = *counts.get("default").unwrap(); + let aux_count = *counts.get("aux").unwrap(); + let roots_count = *counts.get("roots").unwrap(); + let meta_count = *counts.get("meta").unwrap(); + + // Empirical claim: the shielded commitment-tree subtree's ENTIRE state + // — DenseFixedSizedMerkleTree buffer nodes (2-byte BE position keys), + // chunk MMR nodes, META, and the Sinsemilla frontier + // (`COMMITMENT_TREE_DATA_KEY`) — lives in the **default CF only**. The + // frontier is written via `put(...)` not `put_aux(...)` despite the + // misleading internal naming. + // + // This is a simplification of the original snapshot design which + // assumed default + aux. With only default CF in scope, the snapshot + // format reduces to "one SST per subtree" not "two SSTs per subtree". + assert!( + default_count > 0, + "default CF must have entries under subtree prefix; \ + snapshot design is broken if this fails" + ); + assert_eq!( + aux_count, 0, + "aux CF unexpectedly has {aux_count} entries — the Sinsemilla frontier \ + was relocated to aux somewhere. Re-verify storage layout in \ + grovedb-commitment-tree::commitment_tree::mod.rs (search for `put_aux`)" + ); + assert_eq!( + roots_count, 0, + "roots CF unexpectedly has {roots_count} entries under shielded subtree prefix; \ + snapshot dump must include the roots CF (currently doesn't)" + ); + assert_eq!( + meta_count, 0, + "meta CF unexpectedly has {meta_count} entries under shielded subtree prefix; \ + snapshot dump must include the meta CF (currently doesn't)" + ); + } + + /// End-to-end roundtrip: dump the shielded subtree from platform A into a + /// snapshot file, apply it to a fresh platform B, assert anchor_B == + /// anchor_A. Proves that `dump_shielded_subtree` + `apply_shielded_snapshot` + /// preserve the Sinsemilla anchor (which is what wallet sync reads to + /// verify membership proofs). + /// + /// Uses N=30_000 — three batches at `BATCH_SIZE = 10_000`, so the + /// seeder exercises the inter-batch MMR-overlay handoff. The upstream + /// fix `1340db71 fix(commitment-tree): don't commit_mmr inside + /// append_many_raw` (plus our matching loop restructure that defers + /// `commit_mmr` to the end) is what makes multi-batch work; this test + /// pins that contract. + #[test] + #[ignore = "snapshot dump+apply roundtrip; needs the new grovedb branch"] + fn snapshot_dump_apply_preserves_anchor() { + let platform_version = PlatformVersion::latest(); + + // --- A: build, seed, capture anchor --- + let platform_a = build_regtest_platform(); + let cfg = ShieldedSeedConfig { + total_notes: 30_000, + rng_seed: 0xDEAD_BEEF, + }; + let tx_a = platform_a.drive.grove.start_transaction(); + platform_a + .seed_shielded_pool_with_config( + &cfg, + &BlockInfo::default(), + Some(&tx_a), + platform_version, + ) + .expect("seed A"); + tx_a.commit().expect("commit A"); + let anchor_a = read_current_anchor(&platform_a, None, platform_version); + assert_ne!( + anchor_a, EMPTY_SINSEMILLA_ROOT, + "A must have non-empty anchor" + ); + eprintln!("anchor_a = {}", hex::encode(anchor_a)); + + // --- Dump A to a temporary snapshot file --- + let dump_dir = tempfile::tempdir().expect("tempdir"); + let snapshot_path = dump_dir.path().join("shielded-pool.snap"); + let dump_stats = crate::shielded_snapshot::dump_shielded_subtree( + &platform_a.drive.grove, + None, + &snapshot_path, + platform_version, + ) + .expect("dump"); + eprintln!( + "dump: total_count={} key_count={} sst_bytes={}", + dump_stats.total_count, dump_stats.key_count, dump_stats.sst_bytes, + ); + assert_eq!(dump_stats.total_count, u64::from(cfg.total_notes)); + // The DenseFixedSizedMerkleTree + chunk MMR + frontier produces O(N) + // keys; for N=5000 we expect roughly 2000+ keys (sanity: must be > 0). + assert!(dump_stats.key_count > 0); + + // --- B: build (parent skeleton present, shielded pool EMPTY) --- + let platform_b = build_regtest_platform(); + let pre_anchor_b = read_current_anchor(&platform_b, None, platform_version); + assert_eq!( + pre_anchor_b, EMPTY_SINSEMILLA_ROOT, + "B should start with empty shielded pool" + ); + + // --- Apply the snapshot to B --- + let apply_stats = crate::shielded_snapshot::apply_shielded_snapshot( + &platform_b.drive.grove, + None, + &snapshot_path, + platform_version, + ) + .expect("apply"); + eprintln!( + "apply: total_count={} combined_root={}", + apply_stats.total_count, + hex::encode(apply_stats.combined_root), + ); + + // --- Verify: anchor_B equals anchor_A --- + let anchor_b = read_current_anchor(&platform_b, None, platform_version); + eprintln!("anchor_b = {}", hex::encode(anchor_b)); + assert_eq!( + anchor_b, anchor_a, + "anchor must match after dump + apply roundtrip" + ); + + // --- Also verify: total_count via shielded_pool_notes_count --- + let mut drive_ops = vec![]; + let count_b = platform_b + .drive + .shielded_pool_notes_count(None, &mut drive_ops, platform_version) + .expect("count B"); + assert_eq!( + count_b, + u64::from(cfg.total_notes), + "B's note count must match A's seed config" + ); + } + + // Real InitChain hook coverage happens via the dashmate devnet flow + // (see docs/genesis-snapshot-design.md §13 e2e). An in-process equivalent + // would need `std::env::set_var`, which this crate's + // `#![forbid(unsafe_code)]` rejects. The roundtrip test + // `snapshot_dump_apply_preserves_anchor` proves dump+apply preserves the + // anchor; the hook in `create_data_for_shielded_pool` is then a trivial + // env-var read + delegation. +} diff --git a/packages/rs-drive-abci/src/lib.rs b/packages/rs-drive-abci/src/lib.rs index f940b63431e..756e0176bf5 100644 --- a/packages/rs-drive-abci/src/lib.rs +++ b/packages/rs-drive-abci/src/lib.rs @@ -88,5 +88,14 @@ pub mod utils; pub mod replay; /// Drive server pub mod server; +/// Shielded-pool genesis snapshot — bake/apply. +/// +/// Test-data tooling only: the bake reads a pool seeded by +/// `create_sdk_test_data` and the apply runs from that same seeder's +/// fast-path, so the module has no purpose in a production build. Gated on +/// `create_sdk_test_data` (its only callers) plus `test` (the genesis +/// seeder's test module exercises it under `cargo test`). +#[cfg(any(create_sdk_test_data, test))] +pub mod shielded_snapshot; /// Verification helpers pub mod verify; diff --git a/packages/rs-drive-abci/src/main.rs b/packages/rs-drive-abci/src/main.rs index a003e2e2f44..fb84eeefd36 100644 --- a/packages/rs-drive-abci/src/main.rs +++ b/packages/rs-drive-abci/src/main.rs @@ -69,6 +69,24 @@ enum Commands { #[cfg(feature = "replay")] #[command()] Replay(ReplayArgs), + + /// Produce a shielded-pool snapshot file at `--out` by running the full + /// genesis + seed cycle against a fresh temporary GroveDB, then dumping + /// the resulting subtree. Self-contained — does not need a running + /// drive-abci or a populated DB. + /// + /// Intended for the Dockerfile bake stage, where the snapshot file is + /// embedded into the runtime image and consumed at boot via + /// `DRIVE_SHIELDED_SNAPSHOT=`. Requires the binary to be built + /// with `--cfg create_sdk_test_data` so that `create_genesis_state` + /// invokes the seeder — the command is compiled out otherwise. + #[cfg(create_sdk_test_data)] + #[command()] + SnapshotBake { + /// Where to write the snapshot file. Parent directory must exist. + #[arg(long)] + out: PathBuf, + }, } /// Server that accepts connections from Tenderdash, and @@ -167,6 +185,8 @@ impl Cli { Commands::Config => dump_config(&config)?, Commands::Status => runtime.block_on(check_status(&config))?, Commands::Verify => drive_abci::verify::run(&config, true)?, + #[cfg(create_sdk_test_data)] + Commands::SnapshotBake { out } => snapshot_bake(&config, &out)?, Commands::Version => print_version(), #[cfg(feature = "replay")] Commands::Replay(args) => { @@ -181,6 +201,18 @@ impl Cli { fn main() -> Result<(), ExitCode> { let cli = Cli::parse(); + // SnapshotBake runs against an in-container tempdir with no chain env — + // skip `load_config` (which would panic on missing GRPC_BIND_ADDRESS etc.) + // and use a sensible default. Other subcommands (Start / Status / etc.) + // still need the full config. The command only exists under + // `create_sdk_test_data`, so the branch is compiled out otherwise. + #[cfg(create_sdk_test_data)] + let config = if matches!(cli.command, Commands::SnapshotBake { .. }) { + drive_abci::config::PlatformConfig::default_local() + } else { + load_config(&cli.config) + }; + #[cfg(not(create_sdk_test_data))] let config = load_config(&cli.config); // Start tokio runtime and thread listening for signals. @@ -314,6 +346,199 @@ fn dump_config(config: &PlatformConfig) -> Result<(), String> { Ok(()) } +/// Stub CoreRPCLike used during `snapshot-bake` — Platform::open_with_client +/// requires a CoreRPCLike, but create_genesis_state never actually calls any +/// of its methods (no chain locks, transactions, or quorum lookups happen +/// during genesis). Every method is `unreachable!()` so a bake that +/// accidentally tries to talk to Core surfaces as a loud panic. +/// +/// Only the `snapshot-bake` command uses it, so it is compiled out unless +/// `create_sdk_test_data` is set (the cfg that command lives behind). +#[cfg(create_sdk_test_data)] +struct NoopCoreRPC; + +#[cfg(create_sdk_test_data)] +mod noop_core_rpc_impl { + use super::NoopCoreRPC; + use dpp::dashcore::ephemerealdata::chain_lock::ChainLock; + use dpp::dashcore::{Block, BlockHash, Header, InstantLock, QuorumHash, Transaction, Txid}; + use dpp::dashcore_rpc::dashcore_rpc_json::{ + AssetUnlockStatusResult, ExtendedQuorumListResult, GetChainTipsResult, MasternodeListDiff, + MnSyncStatus, QuorumInfoResult, QuorumType, SoftforkInfo, + }; + use dpp::dashcore_rpc::json::GetRawTransactionResult; + use dpp::dashcore_rpc::Error; + use dpp::prelude::TimestampMillis; + use drive_abci::rpc::core::CoreRPCLike; + use serde_json::Value; + + impl CoreRPCLike for NoopCoreRPC { + fn get_block_hash(&self, _: u32) -> Result { + unreachable!() + } + fn get_block_header(&self, _: &BlockHash) -> Result { + unreachable!() + } + fn get_block_time_from_height(&self, _: u32) -> Result { + unreachable!() + } + fn get_best_chain_lock(&self) -> Result { + unreachable!() + } + fn submit_chain_lock(&self, _: &ChainLock) -> Result { + unreachable!() + } + fn get_transaction(&self, _: &Txid) -> Result { + unreachable!() + } + fn get_asset_unlock_statuses( + &self, + _: &[u64], + _: u32, + ) -> Result, Error> { + unreachable!() + } + fn get_transaction_extended_info( + &self, + _: &Txid, + ) -> Result { + unreachable!() + } + fn get_fork_info(&self, _: &str) -> Result, Error> { + unreachable!() + } + fn get_block(&self, _: &BlockHash) -> Result { + unreachable!() + } + fn get_block_json(&self, _: &BlockHash) -> Result { + unreachable!() + } + fn get_chain_tips(&self) -> Result { + unreachable!() + } + fn get_quorum_listextended( + &self, + _: Option, + ) -> Result { + unreachable!() + } + fn get_quorum_info( + &self, + _: QuorumType, + _: &QuorumHash, + _: Option, + ) -> Result { + unreachable!() + } + fn get_protx_diff_with_masternodes( + &self, + _: Option, + _: u32, + ) -> Result { + unreachable!() + } + fn verify_instant_lock(&self, _: &InstantLock, _: Option) -> Result { + unreachable!() + } + fn verify_chain_lock(&self, _: &ChainLock) -> Result { + unreachable!() + } + fn masternode_sync_status(&self) -> Result { + unreachable!() + } + fn send_raw_transaction(&self, _: &[u8]) -> Result { + unreachable!() + } + } +} + +/// Produce a shielded-pool snapshot at `out_path` from a fresh temporary +/// GroveDB. Runs the full `create_genesis_state` cycle (which, under +/// `cfg(create_sdk_test_data)`, invokes the shielded-pool seeder), then +/// dumps the resulting subtree. Self-contained — `config` is ignored +/// (we use a tempdir + sensible defaults). +/// +/// Intended for the Dockerfile bake stage: produce a snapshot once during +/// image build, embed in the runtime image, load it at every InitChain via +/// `DRIVE_SHIELDED_SNAPSHOT`. +#[cfg(create_sdk_test_data)] +fn snapshot_bake(_config: &PlatformConfig, out_path: &std::path::Path) -> Result<(), String> { + use dpp::version::PlatformVersion; + use drive_abci::config::PlatformConfig; + use drive_abci::platform_types::platform::Platform; + + tracing::info!( + out = %out_path.display(), + "snapshot-bake: creating tempdir + bootstrapping fresh GroveDB", + ); + + let tempdir = tempfile::tempdir().map_err(|e| format!("tempdir: {e}"))?; + + // Use the local (regtest) config — same network the bake target image + // will run on. We use NoopCoreRPC so we don't try to connect to a + // non-existent Core node during the in-container bake. + let mut platform_config = PlatformConfig::default_local(); + platform_config.db_path = tempdir.path().to_path_buf(); + + let platform = Platform::::open_with_client( + tempdir.path(), + Some(platform_config), + NoopCoreRPC, + None, + ) + .map_err(|e| format!("open platform: {e}"))?; + + let platform_version = PlatformVersion::latest(); + let tx = platform.drive.grove.start_transaction(); + + // Defensively unset DRIVE_SHIELDED_SNAPSHOT before seeding. The seeder + // (`create_data_for_shielded_pool`) checks this env var first and, if set, + // APPLIES the referenced snapshot instead of running the seeder. A developer + // (or the Dockerfile env) with it exported would make `snapshot-bake` + // recursively re-dump an inherited snapshot rather than seeding a fresh one. + std::env::remove_var("DRIVE_SHIELDED_SNAPSHOT"); + + tracing::info!("snapshot-bake: running create_genesis_state (seeds shielded pool under cfg(create_sdk_test_data))"); + platform + .create_genesis_state( + 1, // genesis_core_height (placeholder for bake) + 0, // genesis_time (placeholder for bake) + Some(&tx), + platform_version, + ) + .map_err(|e| format!("create_genesis_state: {e}"))?; + tx.commit().map_err(|e| format!("commit: {e}"))?; + + tracing::info!( + out = %out_path.display(), + "snapshot-bake: dumping shielded subtree to snapshot file", + ); + let stats = drive_abci::shielded_snapshot::dump_shielded_subtree( + &platform.drive.grove, + None, + out_path, + platform_version, + ) + .map_err(|e| format!("snapshot dump failed: {e}"))?; + + tracing::info!( + out = %out_path.display(), + total_count = stats.total_count, + key_count = stats.key_count, + sst_bytes = stats.sst_bytes, + "snapshot-bake: wrote shielded-pool snapshot", + ); + println!( + "wrote {} bytes ({} keys, total_count={}) to {}", + stats.sst_bytes, + stats.key_count, + stats.total_count, + out_path.display(), + ); + + Ok(()) +} + fn list_enabled_features() -> Vec<&'static str> { vec![ #[cfg(feature = "console")] diff --git a/packages/rs-drive-abci/src/shielded_snapshot/mod.rs b/packages/rs-drive-abci/src/shielded_snapshot/mod.rs new file mode 100644 index 00000000000..82fbf1c339a --- /dev/null +++ b/packages/rs-drive-abci/src/shielded_snapshot/mod.rs @@ -0,0 +1,660 @@ +//! Shielded-pool genesis snapshot — production module. +//! +//! Reduces shielded-pool seeding from a runtime cost (~3h 41m for 500k notes +//! on macOS Docker, ~65 min native) to a one-shot offline bake + at-boot +//! SST ingest (~few seconds at any N). +//! +//! Two entry points: +//! +//! - [`dump_shielded_subtree`] — runs from a snapshot-bake binary. Reads the +//! already-populated shielded subtree from a live GroveDB and writes a +//! portable snapshot file containing one SST blob + header + checksum. +//! - [`apply_shielded_snapshot`] — runs from drive-abci's `InitChain`. Reads +//! the snapshot file, validates header + checksum, ingests the SST into +//! the underlying RocksDB, cross-validates the reconstructed state against +//! the header's `combined_root`, then writes the parent-Merk +//! `Element::CommitmentTree` leaf via the new +//! `GroveDb::replace_commitment_tree_subtree_root` public API. +//! +//! Built on three new public methods we added to grovedb on the +//! `feat/snapshot-apply-public-api` branch: +//! +//! 1. `GroveDb::raw_storage()` — escape hatch to the underlying +//! `RocksDbStorage` so we can open a `StorageContext` for raw iteration. +//! 2. `GroveDb::ingest_subtree_sst(cf, sst_path)` — bulk-ingest an SST file. +//! 3. `GroveDb::replace_commitment_tree_subtree_root(...)` — patch the +//! parent-Merk `Element::CommitmentTree` leaf with a caller-provided +//! `combined_root`. +//! +//! See `docs/genesis-snapshot-design.md` for the full design + threat model. + +#![allow(missing_docs)] + +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use dpp::version::PlatformVersion; +use drive::drive::shielded::paths::{shielded_credit_pool_path_vec, SHIELDED_NOTES_KEY}; +use drive::grovedb::{Element, GroveDb, TransactionArg}; +use drive::grovedb_path::SubtreePath; +use drive::grovedb_storage::rocksdb_storage::RocksDbStorage; +use drive::grovedb_storage::{RawIterator, Storage, StorageContext}; +use grovedb_commitment_tree::{CommitmentTree, DashMemo}; + +/// File magic — 8 bytes, matches the `b"DRVSHLD\0"` literal. +const MAGIC: [u8; 8] = *b"DRVSHLD\0"; +/// Snapshot file format version. Bump on any breaking header/section change. +/// v2: parent-leaf flags encoded as `[present, byte]` (injective `Option`) +/// instead of a single byte that collapsed `None` and `Some(0)`. +const FORMAT_VERSION: u32 = 2; + +/// Max permitted `chunk_power`. BulkAppendTree internally caps at 31; for +/// genesis snapshots we want a tighter sanity bound — anything > 16 would +/// imply a chunk holding ≥ 65 536 cmx values, which is far beyond anything +/// we'd ever ship. +const MAX_CHUNK_POWER: u8 = 16; + +/// CF where the shielded subtree's keys live. Pinned empirically by the +/// `dump_only_default_and_aux_cfs_under_shielded_subtree_prefix` test — +/// EVERY key (BulkAppendTree state, dense-tree buffer, MMR nodes, META, +/// Sinsemilla frontier) is in the default CF for this subtree. +const SUBTREE_CF: &str = rocksdb::DEFAULT_COLUMN_FAMILY_NAME; + +/// Errors surfaced by [`dump_shielded_subtree`] and [`apply_shielded_snapshot`]. +#[derive(Debug)] +pub enum ShieldedSnapshotError { + Io(std::io::Error), + InvalidMagic { + got: [u8; 8], + }, + FormatVersionMismatch { + expected: u32, + found: u32, + }, + ChunkPowerTooLarge { + got: u8, + max: u8, + }, + ChecksumMismatch { + expected: [u8; 32], + computed: [u8; 32], + }, + /// Header's `combined_root` doesn't match what reconstructing the + /// CommitmentTree from the ingested data produces. Indicates tampering, + /// truncation, or version skew. + CombinedRootMismatch { + expected: [u8; 32], + computed: [u8; 32], + }, + /// The element at the expected parent-leaf path/key is not + /// `Element::CommitmentTree`. InitChain must build the parent skeleton + /// before applying the snapshot. + ParentLeafWrongType, + /// Bubbled from GroveDB / grovedb-storage / grovedb-commitment-tree. + GroveDb(String), + /// Bubbled from rocksdb (SST writer, ingest). + RocksDb(String), + /// Inconsistent header/file state. + Inconsistent(String), +} + +impl std::fmt::Display for ShieldedSnapshotError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Io(e) => write!(f, "i/o: {e}"), + Self::InvalidMagic { got } => write!(f, "invalid magic: {got:?}"), + Self::FormatVersionMismatch { expected, found } => { + write!( + f, + "format_version mismatch (expected {expected}, got {found})" + ) + } + Self::ChunkPowerTooLarge { got, max } => { + write!(f, "chunk_power {got} exceeds max {max}") + } + Self::ChecksumMismatch { .. } => write!(f, "checksum mismatch"), + Self::CombinedRootMismatch { .. } => write!( + f, + "combined_root mismatch — snapshot data doesn't match header" + ), + Self::ParentLeafWrongType => write!(f, "parent leaf is not Element::CommitmentTree"), + Self::GroveDb(s) => write!(f, "grovedb: {s}"), + Self::RocksDb(s) => write!(f, "rocksdb: {s}"), + Self::Inconsistent(s) => write!(f, "inconsistent: {s}"), + } + } +} + +impl std::error::Error for ShieldedSnapshotError {} + +impl From for ShieldedSnapshotError { + fn from(e: std::io::Error) -> Self { + Self::Io(e) + } +} + +/// Header of a shielded-pool snapshot file. See module docs for fields. +#[derive(Debug, Clone)] +pub struct SnapshotHeader { + pub format_version: u32, + pub total_count: u64, + pub chunk_power: u8, + /// Parent-leaf `Element::CommitmentTree` flags (`Option` + /// where `ElementFlags = Vec`), narrowed to "absent, or exactly one + /// byte" — the only shapes the shielded leaf carries in practice. + /// + /// Encoded as a presence discriminator byte plus the value byte so the + /// round-trip is injective: `None` and `Some(0)` are distinct on the + /// wire (a single `flags_byte` collapsed both to `0`, silently rewriting + /// `Some(vec![0])` as `None` and diverging the parent-Merk serialization + /// from a live-seeded tree even though `combined_root` still matched). + /// The dumper rejects an empty or multi-byte flags vec loudly; widening + /// past one byte requires bumping `FORMAT_VERSION`. + pub flags: Option, + pub combined_root: [u8; 32], + pub sst_len: u64, +} + +impl SnapshotHeader { + /// Wire size of the encoded header (NOT including the SST blob or the + /// trailing checksum). + // flags is encoded as 2 bytes: [present, byte]. + const ENCODED_LEN: usize = 8 + 4 + 8 + 1 + 2 + 32 + 8; // = 63 bytes + + fn write_to(&self, w: &mut W) -> Result<(), std::io::Error> { + w.write_all(&MAGIC)?; + w.write_all(&self.format_version.to_be_bytes())?; + w.write_all(&self.total_count.to_be_bytes())?; + w.write_all(&[self.chunk_power])?; + // [present, byte]: present=1 means Some(byte); present=0 means None + // (byte then ignored). Keeps None distinct from Some(0). + let (present, byte) = match self.flags { + Some(b) => (1u8, b), + None => (0u8, 0u8), + }; + w.write_all(&[present, byte])?; + w.write_all(&self.combined_root)?; + w.write_all(&self.sst_len.to_be_bytes())?; + Ok(()) + } + + fn read_from(r: &mut R) -> Result { + let mut magic = [0u8; 8]; + r.read_exact(&mut magic)?; + if magic != MAGIC { + return Err(ShieldedSnapshotError::InvalidMagic { got: magic }); + } + let mut buf4 = [0u8; 4]; + r.read_exact(&mut buf4)?; + let format_version = u32::from_be_bytes(buf4); + if format_version != FORMAT_VERSION { + return Err(ShieldedSnapshotError::FormatVersionMismatch { + expected: FORMAT_VERSION, + found: format_version, + }); + } + let mut buf8 = [0u8; 8]; + r.read_exact(&mut buf8)?; + let total_count = u64::from_be_bytes(buf8); + let mut one = [0u8; 1]; + r.read_exact(&mut one)?; + let chunk_power = one[0]; + if chunk_power > MAX_CHUNK_POWER { + return Err(ShieldedSnapshotError::ChunkPowerTooLarge { + got: chunk_power, + max: MAX_CHUNK_POWER, + }); + } + let mut two = [0u8; 2]; + r.read_exact(&mut two)?; + let flags = match two[0] { + 0 => None, + 1 => Some(two[1]), + other => { + return Err(ShieldedSnapshotError::Inconsistent(format!( + "invalid flags presence discriminator {other}; expected 0 (None) or 1 (Some)" + ))); + } + }; + let mut combined_root = [0u8; 32]; + r.read_exact(&mut combined_root)?; + r.read_exact(&mut buf8)?; + let sst_len = u64::from_be_bytes(buf8); + Ok(Self { + format_version, + total_count, + chunk_power, + flags, + combined_root, + sst_len, + }) + } +} + +/// Stats returned by [`dump_shielded_subtree`]. +#[derive(Debug, Clone)] +pub struct DumpStats { + pub total_count: u64, + pub key_count: u64, + pub sst_bytes: u64, +} + +/// Stats returned by [`apply_shielded_snapshot`]. +#[derive(Debug, Clone)] +pub struct ApplyStats { + pub total_count: u64, + pub combined_root: [u8; 32], +} + +/// Build the SubtreePath segments for the shielded commitment-tree subtree. +fn shielded_subtree_segments() -> Vec> { + let mut v = shielded_credit_pool_path_vec(); + v.push(vec![SHIELDED_NOTES_KEY]); + v +} + +/// Iterate the live shielded subtree's keys and write them into a snapshot +/// file at `out_path`. +/// +/// Uses the public [`GroveDb`] API + raw RocksDB `SstFileWriter` to produce +/// a snapshot the apply side can `ingest_external_file_cf` into a fresh DB. +/// +/// Reads the parent-Merk `Element::CommitmentTree` leaf to populate the +/// header's `total_count`/`chunk_power`/`flags`. Computes `combined_root` by +/// reconstructing the CommitmentTree from the same storage we're about to +/// dump — that root is what the apply side cross-validates against. +pub fn dump_shielded_subtree( + grove: &GroveDb, + transaction: TransactionArg, + out_path: &Path, + platform_version: &PlatformVersion, +) -> Result { + use rocksdb::{Options, SstFileWriter}; + + let parent_segments = shielded_credit_pool_path_vec(); + let parent_path = SubtreePath::from(parent_segments.as_slice()); + let leaf_key = &[SHIELDED_NOTES_KEY]; + + // 1. Read parent leaf for header values. + let element = grove + .get_raw( + parent_path, + leaf_key, + transaction, + &platform_version.drive.grove_version, + ) + .value + .map_err(|e| ShieldedSnapshotError::GroveDb(format!("get_raw parent leaf: {e}")))?; + let (total_count, chunk_power, flags) = match element { + Element::CommitmentTree(tc, cp, f) => (tc, cp, f), + _ => return Err(ShieldedSnapshotError::ParentLeafWrongType), + }; + if chunk_power > MAX_CHUNK_POWER { + return Err(ShieldedSnapshotError::ChunkPowerTooLarge { + got: chunk_power, + max: MAX_CHUNK_POWER, + }); + } + + // 2. Compute the 32-byte subtree prefix. RocksDB SST keys must be FULL + // keys (prefix prepended) because ingest_external_file_cf doesn't + // prepend anything — it expects already-final bytes. + let subtree_segments = shielded_subtree_segments(); + let subtree_path = SubtreePath::from(subtree_segments.as_slice()); + let prefix: [u8; 32] = RocksDbStorage::build_prefix(subtree_path.clone()).unwrap(); + + // 3. Open transactional storage context at the subtree path. We use the + // caller's transaction if provided; otherwise start a local one. + // The context's raw_iter strips the prefix; we re-add it for the SST. + let local_tx; + let tx_ref: &drive::grovedb::Transaction = match transaction { + Some(t) => t, + None => { + local_tx = grove.start_transaction(); + &local_tx + } + }; + let storage_ctx = grove + .raw_storage() + .get_transactional_storage_context(subtree_path, None, tx_ref) + .unwrap(); + + // 4. Compute `combined_root` for the header by reloading CommitmentTree + // from the same storage. Apply side recomputes independently and + // cross-validates — drift surfaces as CombinedRootMismatch. + let ct = CommitmentTree::<_, DashMemo>::open(total_count, chunk_power, storage_ctx) + .value + .map_err(|e| ShieldedSnapshotError::GroveDb(format!("CommitmentTree::open: {e}")))?; + let combined_root = ct + .compute_current_state_root() + .map_err(|e| ShieldedSnapshotError::GroveDb(format!("compute_current_state_root: {e}")))?; + + // The CommitmentTree owns the storage_ctx. We need to drop it to free + // the borrow before opening the iterator on a fresh context. + drop(ct); + + // 5. Open a SECOND storage context just for iteration. Two contexts on + // the same path/txn is safe — they share the underlying transaction. + let subtree_segments_for_iter = shielded_subtree_segments(); + let iter_path = SubtreePath::from(subtree_segments_for_iter.as_slice()); + let iter_ctx = grove + .raw_storage() + .get_transactional_storage_context(iter_path, None, tx_ref) + .unwrap(); + + // 6. Open SST writer. Write SST to tmp file alongside the output path. + let tmp_dir = out_path + .parent() + .unwrap_or_else(|| Path::new(".")) + .to_path_buf(); + let sst_tmp = tmp_dir.join(format!( + ".{}.sst.tmp", + out_path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("shielded-snapshot") + )); + let sst_opts = Options::default(); + let mut sst = SstFileWriter::create(&sst_opts); + sst.open(&sst_tmp) + .map_err(|e| ShieldedSnapshotError::RocksDb(format!("SstFileWriter::open: {e}")))?; + + // 7. Walk subtree keys (sorted by RocksDB iterator), prepend prefix, + // write to SST. RocksDB SST requires keys in strictly increasing + // order — the raw_iter returns keys in lex order so prefix-prepended + // keys are also lex-ordered. + let mut iter = iter_ctx.raw_iter(); + iter.seek_to_first().unwrap(); + let mut key_count: u64 = 0; + loop { + if !iter.valid().unwrap() { + break; + } + let user_key = match iter.key().unwrap() { + Some(k) => k.to_vec(), + None => break, + }; + let value = match iter.value().unwrap() { + Some(v) => v.to_vec(), + None => break, + }; + let mut full_key = Vec::with_capacity(32 + user_key.len()); + full_key.extend_from_slice(&prefix); + full_key.extend_from_slice(&user_key); + sst.put(&full_key, &value) + .map_err(|e| ShieldedSnapshotError::RocksDb(format!("SstFileWriter::put: {e}")))?; + key_count += 1; + iter.next().unwrap(); + } + sst.finish() + .map_err(|e| ShieldedSnapshotError::RocksDb(format!("SstFileWriter::finish: {e}")))?; + let sst_bytes_on_disk = std::fs::metadata(&sst_tmp)?.len(); + + // 8. Compose the output file: header || sst_bytes || blake3 checksum. + // + // The header encodes the parent-leaf flags as `Option` — absent, or + // exactly one byte — preserving `None` vs `Some(0)` so a snapshot-booted + // devnet reconstructs the identical parent-Merk leaf. Fail loudly on any + // other shape (empty or multi-byte vec) rather than lossily folding it: + // either would diverge from a seeder-built tree on the parent-Merk root + // even though `combined_root` (which covers only the rebuilt subtree) + // still matches. + let flags_opt: Option = match flags.as_deref() { + None => None, + Some([b]) => Some(*b), + Some(v) => { + return Err(ShieldedSnapshotError::Inconsistent(format!( + "parent-leaf flags has {} bytes; snapshot format encodes None or exactly 1 byte — bump FORMAT_VERSION and widen the header", + v.len() + ))); + } + }; + let header = SnapshotHeader { + format_version: FORMAT_VERSION, + total_count, + chunk_power, + flags: flags_opt, + combined_root, + sst_len: sst_bytes_on_disk, + }; + + let mut hasher = blake3::Hasher::new(); + let mut out = std::fs::File::create(out_path)?; + let mut header_buf = Vec::with_capacity(SnapshotHeader::ENCODED_LEN); + header.write_to(&mut header_buf)?; + hasher.update(&header_buf); + out.write_all(&header_buf)?; + + // Stream the SST file contents in chunks to avoid loading the full + // (potentially-huge) SST into RAM. + let mut sst_file = std::fs::File::open(&sst_tmp)?; + let mut buf = vec![0u8; 256 * 1024]; + loop { + let n = sst_file.read(&mut buf)?; + if n == 0 { + break; + } + hasher.update(&buf[..n]); + out.write_all(&buf[..n])?; + } + + let checksum = hasher.finalize(); + out.write_all(checksum.as_bytes())?; + out.sync_all()?; + + let _ = std::fs::remove_file(&sst_tmp); + + Ok(DumpStats { + total_count, + key_count, + sst_bytes: sst_bytes_on_disk, + }) +} + +/// Apply a snapshot file produced by [`dump_shielded_subtree`] into a fresh +/// GroveDB. Intended to be called from `InitChain` AFTER the parent tree +/// skeleton (`/platform/shielded_pool/`) has been built but BEFORE the +/// shielded subtree would otherwise be populated by the runtime seeder. +pub fn apply_shielded_snapshot( + grove: &GroveDb, + transaction: TransactionArg, + snapshot_path: &Path, + platform_version: &PlatformVersion, +) -> Result { + // 1. Read the file into memory (small enough for N up to ~500k). + let bytes = std::fs::read(snapshot_path)?; + if bytes.len() < SnapshotHeader::ENCODED_LEN + 32 { + return Err(ShieldedSnapshotError::Inconsistent(format!( + "snapshot file shorter than minimal envelope ({} bytes)", + bytes.len() + ))); + } + + // 2. Parse + verify checksum. + let header = { + let mut cursor = std::io::Cursor::new(&bytes[..SnapshotHeader::ENCODED_LEN]); + SnapshotHeader::read_from(&mut cursor)? + }; + let body_start = SnapshotHeader::ENCODED_LEN; + let body_end = body_start + header.sst_len as usize; + if body_end + 32 > bytes.len() { + return Err(ShieldedSnapshotError::Inconsistent(format!( + "header says sst_len={} but file truncated", + header.sst_len + ))); + } + let sst_slice = &bytes[body_start..body_end]; + + let mut stored = [0u8; 32]; + stored.copy_from_slice(&bytes[body_end..body_end + 32]); + let mut hasher = blake3::Hasher::new(); + hasher.update(&bytes[..body_end]); + let computed = *hasher.finalize().as_bytes(); + if stored != computed { + return Err(ShieldedSnapshotError::ChecksumMismatch { + expected: stored, + computed, + }); + } + + // 3. Materialise SST to a tmp file (rocksdb ingest needs a real path). + let tmp_dir = std::env::temp_dir(); + let sst_tmp = tmp_dir.join(format!( + "drv-shielded-snapshot-{}-{}.sst", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0), + )); + std::fs::write(&sst_tmp, sst_slice)?; + let _cleanup = SstTmpGuard(sst_tmp.clone()); + + // 4. Bulk-ingest. Bypasses any open transaction; OK at InitChain time + // (txn abort = wipe-and-restart, so orphan data is unreachable). + grove + .ingest_subtree_sst(SUBTREE_CF, &sst_tmp) + .map_err(|e| ShieldedSnapshotError::GroveDb(format!("ingest_subtree_sst: {e}")))?; + + // 5. Cross-validate: reload CommitmentTree from the just-ingested data + // and check recomputed combined_root matches header. Drift surfaces + // BEFORE we touch the parent Merk. + let subtree_segments = shielded_subtree_segments(); + let subtree_path = SubtreePath::from(subtree_segments.as_slice()); + + let local_tx; + let tx_ref: &drive::grovedb::Transaction = match transaction { + Some(t) => t, + None => { + local_tx = grove.start_transaction(); + &local_tx + } + }; + let storage_ctx = grove + .raw_storage() + .get_transactional_storage_context(subtree_path, None, tx_ref) + .unwrap(); + + let ct = + CommitmentTree::<_, DashMemo>::open(header.total_count, header.chunk_power, storage_ctx) + .value + .map_err(|e| { + ShieldedSnapshotError::GroveDb(format!("CommitmentTree::open after ingest: {e}")) + })?; + let recomputed = ct + .compute_current_state_root() + .map_err(|e| ShieldedSnapshotError::GroveDb(format!("compute_current_state_root: {e}")))?; + drop(ct); + + if recomputed != header.combined_root { + return Err(ShieldedSnapshotError::CombinedRootMismatch { + expected: header.combined_root, + computed: recomputed, + }); + } + + // 6. Patch parent Merk leaf. + let parent_segments = shielded_credit_pool_path_vec(); + let parent_path = SubtreePath::from(parent_segments.as_slice()); + let leaf_key = &[SHIELDED_NOTES_KEY]; + let flags = header.flags.map(|b| vec![b]); + + grove + .replace_commitment_tree_subtree_root( + parent_path, + leaf_key, + header.total_count, + header.chunk_power, + flags, + header.combined_root, + transaction, + &platform_version.drive.grove_version, + ) + .value + .map_err(|e| { + ShieldedSnapshotError::GroveDb(format!("replace_commitment_tree_subtree_root: {e}")) + })?; + + Ok(ApplyStats { + total_count: header.total_count, + combined_root: header.combined_root, + }) +} + +/// RAII guard that removes the apply-side tmp SST file regardless of which +/// path through the function we exit on. +struct SstTmpGuard(PathBuf); +impl Drop for SstTmpGuard { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Encode a header with the given parent-leaf flags, decode it back, and + /// return the decoded `flags` so a test can assert the round-trip. + fn roundtrip_flags(flags: Option) -> Option { + let header = SnapshotHeader { + format_version: FORMAT_VERSION, + total_count: 7, + chunk_power: 11, + flags, + combined_root: [0xAB; 32], + sst_len: 123, + }; + let mut buf = Vec::with_capacity(SnapshotHeader::ENCODED_LEN); + header.write_to(&mut buf).expect("write_to"); + assert_eq!( + buf.len(), + SnapshotHeader::ENCODED_LEN, + "encoded header length must match ENCODED_LEN" + ); + let mut cursor = std::io::Cursor::new(buf); + SnapshotHeader::read_from(&mut cursor) + .expect("read_from") + .flags + } + + /// The flags round-trip must be injective: `None` and `Some(0)` are + /// distinct on the wire. A prior single-byte encoding collapsed both to + /// `0`/`None`, silently rewriting `Some(vec![0])` as `None` on apply. + #[test] + fn flags_roundtrip_is_injective() { + assert_eq!(roundtrip_flags(None), None, "None must stay None"); + assert_eq!( + roundtrip_flags(Some(0)), + Some(0), + "Some(0) must stay Some(0), not collapse to None" + ); + assert_eq!(roundtrip_flags(Some(0xAB)), Some(0xAB)); + // The two cases that previously aliased are now distinguishable. + assert_ne!(roundtrip_flags(None), roundtrip_flags(Some(0))); + } + + /// An invalid presence discriminator (anything but 0 or 1) is rejected. + #[test] + fn flags_invalid_presence_discriminator_rejected() { + let header = SnapshotHeader { + format_version: FORMAT_VERSION, + total_count: 0, + chunk_power: 0, + flags: None, + combined_root: [0u8; 32], + sst_len: 0, + }; + let mut buf = Vec::new(); + header.write_to(&mut buf).expect("write_to"); + // Corrupt the presence byte (immediately after MAGIC + version + + // total_count + chunk_power = 8 + 4 + 8 + 1 = offset 21). + buf[21] = 2; + let mut cursor = std::io::Cursor::new(buf); + let err = SnapshotHeader::read_from(&mut cursor).unwrap_err(); + assert!( + matches!(err, ShieldedSnapshotError::Inconsistent(_)), + "expected Inconsistent for bad presence discriminator, got {err:?}" + ); + } +} diff --git a/packages/rs-drive-proof-verifier/src/proof.rs b/packages/rs-drive-proof-verifier/src/proof.rs index 3cf070c266e..d685685248b 100644 --- a/packages/rs-drive-proof-verifier/src/proof.rs +++ b/packages/rs-drive-proof-verifier/src/proof.rs @@ -2589,7 +2589,7 @@ impl FromProof for ShieldedEncrypted .max_query_chunks as u32 * (1u32 << drive::drive::shielded::paths::SHIELDED_NOTES_CHUNK_POWER); - let (root_hash, notes) = Drive::verify_shielded_encrypted_notes( + let (root_hash, notes, total_count) = Drive::verify_shielded_encrypted_notes( &proof.grovedb_proof, start_index, count, @@ -2601,20 +2601,22 @@ impl FromProof for ShieldedEncrypted verify_tenderdash_proof(proof, mtd, &root_hash, provider)?; - let result = if notes.is_empty() { - None - } else { - Some(ShieldedEncryptedNotes( - notes - .into_iter() - .map(|(cmx, nullifier, encrypted_note)| ShieldedEncryptedNote { - cmx, - nullifier, - encrypted_note, - }) - .collect(), - )) - }; + // `total_count` (the on-chain total note count) is extracted from the + // same proof, so it is available even when this chunk returned no + // notes — return the result whenever the proof verified, carrying the + // count for the sync progress-bar denominator. `None` would only be + // appropriate if the proof itself were absent, which is handled above. + let result = Some(ShieldedEncryptedNotes { + notes: notes + .into_iter() + .map(|(cmx, nullifier, encrypted_note)| ShieldedEncryptedNote { + cmx, + nullifier, + encrypted_note, + }) + .collect(), + total_count, + }); Ok((result, mtd.clone(), proof.clone())) } diff --git a/packages/rs-drive-proof-verifier/src/types.rs b/packages/rs-drive-proof-verifier/src/types.rs index f389186701c..7922782c5ae 100644 --- a/packages/rs-drive-proof-verifier/src/types.rs +++ b/packages/rs-drive-proof-verifier/src/types.rs @@ -861,14 +861,26 @@ pub struct ShieldedEncryptedNote { pub encrypted_note: Vec, } -/// Collection of encrypted notes returned by query -#[derive(Debug, Clone, Default, derive_more::From)] +/// Collection of encrypted notes returned by query. +/// +/// `total_count` is the on-chain total number of notes in the shielded +/// `CommitmentTree` at the proven block — the denominator a wallet needs +/// for a sync progress bar. It is extracted from the SAME note-fetch proof +/// (the parent CommitmentTree element is always present in that proof), so +/// every chunk fetch carries the total "for free" with no separate RPC. +#[derive(Debug, Clone, Default)] #[cfg_attr( feature = "mocks", derive(Encode, Decode, PlatformSerialize, PlatformDeserialize), platform_serialize(unversioned) )] -pub struct ShieldedEncryptedNotes(pub Vec); +pub struct ShieldedEncryptedNotes { + /// The encrypted notes for the requested chunk, in tree order. + pub notes: Vec, + /// On-chain total number of notes in the shielded `CommitmentTree`. + /// Stable across a sync; carried on every chunk fetch. + pub total_count: u64, +} /// Valid anchors for building spend proofs #[derive(Debug, Clone, Default, derive_more::From)] diff --git a/packages/rs-drive/Cargo.toml b/packages/rs-drive/Cargo.toml index bd4b74577e6..c893ae54b9a 100644 --- a/packages/rs-drive/Cargo.toml +++ b/packages/rs-drive/Cargo.toml @@ -52,12 +52,12 @@ enum-map = { version = "2.0.3", optional = true } intmap = { version = "3.0.1", features = ["serde"], optional = true } chrono = { version = "0.4.35", optional = true } itertools = { version = "0.13", optional = true } -grovedb = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9", optional = true, default-features = false } -grovedb-costs = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9", optional = true } -grovedb-path = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9" } -grovedb-storage = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9", optional = true } -grovedb-version = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9" } -grovedb-epoch-based-storage-flags = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9" } +grovedb = { git = "https://github.com/dashpay/grovedb", rev = "5eb7a5380a6e974513343352acfd6b30a8c1f87c", optional = true, default-features = false } +grovedb-costs = { git = "https://github.com/dashpay/grovedb", rev = "5eb7a5380a6e974513343352acfd6b30a8c1f87c", optional = true } +grovedb-path = { git = "https://github.com/dashpay/grovedb", rev = "5eb7a5380a6e974513343352acfd6b30a8c1f87c" } +grovedb-storage = { git = "https://github.com/dashpay/grovedb", rev = "5eb7a5380a6e974513343352acfd6b30a8c1f87c", optional = true } +grovedb-version = { git = "https://github.com/dashpay/grovedb", rev = "5eb7a5380a6e974513343352acfd6b30a8c1f87c" } +grovedb-epoch-based-storage-flags = { git = "https://github.com/dashpay/grovedb", rev = "5eb7a5380a6e974513343352acfd6b30a8c1f87c" } [dev-dependencies] criterion = "0.5" diff --git a/packages/rs-drive/src/verify/shielded/verify_shielded_encrypted_notes/mod.rs b/packages/rs-drive/src/verify/shielded/verify_shielded_encrypted_notes/mod.rs index aa7ba259043..172a6cecd5d 100644 --- a/packages/rs-drive/src/verify/shielded/verify_shielded_encrypted_notes/mod.rs +++ b/packages/rs-drive/src/verify/shielded/verify_shielded_encrypted_notes/mod.rs @@ -8,6 +8,12 @@ use dpp::version::PlatformVersion; impl Drive { /// Verifies a proof for shielded encrypted notes. + /// + /// Returns `(root_hash, notes, total_count)`. `total_count` is the + /// on-chain total number of notes in the shielded `CommitmentTree`, + /// extracted from the SAME proof (the parent CommitmentTree element is + /// always present in a note-fetch proof) — wallets get the sync + /// progress-bar denominator for free on every chunk fetch. #[allow(clippy::type_complexity)] pub fn verify_shielded_encrypted_notes( proof: &[u8], @@ -16,7 +22,7 @@ impl Drive { max_elements: u32, verify_subset_of_proof: bool, platform_version: &PlatformVersion, - ) -> Result<(RootHash, Vec<(Vec, Vec, Vec)>), Error> { + ) -> Result<(RootHash, Vec<(Vec, Vec, Vec)>, u64), Error> { match platform_version .drive .methods diff --git a/packages/rs-drive/src/verify/shielded/verify_shielded_encrypted_notes/v0/mod.rs b/packages/rs-drive/src/verify/shielded/verify_shielded_encrypted_notes/v0/mod.rs index e40b02861b3..7310d1a0c4e 100644 --- a/packages/rs-drive/src/verify/shielded/verify_shielded_encrypted_notes/v0/mod.rs +++ b/packages/rs-drive/src/verify/shielded/verify_shielded_encrypted_notes/v0/mod.rs @@ -1,12 +1,32 @@ use crate::drive::shielded::paths::{shielded_credit_pool_path_vec, SHIELDED_NOTES_KEY}; use crate::drive::Drive; use crate::error::drive::DriveError; +use crate::error::proof::ProofError; use crate::error::Error; use crate::verify::RootHash; use grovedb::{Element, GroveDb, PathQuery, Query, QueryItem, SizedQuery, SubqueryBranch}; use platform_version::version::PlatformVersion; impl Drive { + /// Verifies a `GetShieldedEncryptedNotes` proof. + /// + /// Returns `(root_hash, notes, total_count)` where `total_count` is the + /// on-chain total number of notes appended to the shielded + /// `CommitmentTree` — i.e. the denominator a wallet needs for a sync + /// progress bar. + /// + /// `total_count` comes "for free" from the same proof: the note-fetch + /// proof is a GroveDB layered proof that subqueries INTO the + /// `CommitmentTree` element at `[ShieldedBalances, "M", [128]]`. That + /// parent element is necessarily present in the proof (it chains to the + /// root hash and supplies `total_count` to the inner `CommitmentTree` + /// subquery verification). We extract it by running an additional + /// **subset** verification against the SAME proof bytes with a + /// single-key `PathQuery` targeting the `CommitmentTree` element itself + /// (no subquery), then decoding `Element::CommitmentTree(total_count, + /// ..)` — field 0. This reuses the exact `PathQuery` + decode the + /// standalone `GetShieldedNotesCount` verifier would use, but against + /// the note-fetch proof we already have. #[allow(clippy::type_complexity)] pub(super) fn verify_shielded_encrypted_notes_v0( proof: &[u8], @@ -15,7 +35,7 @@ impl Drive { max_elements: u32, verify_subset_of_proof: bool, platform_version: &PlatformVersion, - ) -> Result<(RootHash, Vec<(Vec, Vec, Vec)>), Error> { + ) -> Result<(RootHash, Vec<(Vec, Vec, Vec)>, u64), Error> { if max_elements == 0 { return Err(Error::Drive(DriveError::CorruptedElementType( "max_elements must be greater than zero", @@ -98,6 +118,223 @@ impl Drive { } } - Ok((root_hash, notes)) + // Extract the on-chain total note count from the SAME proof bytes. + // + // The note-fetch proof above subqueries INTO the CommitmentTree, so + // the parent `Element::CommitmentTree(total_count, ..)` at key + // `[SHIELDED_NOTES_KEY]` under `[ShieldedBalances, "M"]` is already + // covered by the proof. A single-key query for that element (no + // subquery) is a strict subset of what the note-fetch proof proves, + // so a *subset* verification against the same bytes must succeed. + // `verify_subset_query` is used unconditionally (not gated on + // `verify_subset_of_proof`) because this element query is, by + // construction, a sub-portion of the larger proof. + let count_path_query = PathQuery { + path: shielded_credit_pool_path_vec(), + query: SizedQuery { + query: Query::new_single_key(vec![SHIELDED_NOTES_KEY]), + limit: Some(1), + offset: None, + }, + }; + + let (count_root_hash, mut count_proved_key_values) = GroveDb::verify_subset_query( + proof, + &count_path_query, + &platform_version.drive.grove_version, + )?; + + // Both verifications run against the SAME proof bytes, so they must + // derive the same root by construction. Make that invariant explicit: + // a mismatch means the count sub-proof and the note-fetch proof came + // from different state and the result cannot be trusted. + if count_root_hash != root_hash { + return Err(Error::Proof(ProofError::IncorrectProof( + "shielded notes count sub-proof root mismatch".to_string(), + ))); + } + + let total_count = match count_proved_key_values.pop() { + Some((_, _, Some(Element::CommitmentTree(total_count, _, _)))) => total_count, + Some((_, _, Some(_))) => { + return Err(Error::Drive(DriveError::CorruptedElementType( + "expected CommitmentTree element for shielded notes count, got different element type", + ))); + } + // Absent parent element: an uninitialized pool would lack the + // CommitmentTree leaf. A valid note-fetch proof against a live + // pool always carries it; treat absence as zero notes. + Some((_, _, None)) | None => 0, + }; + + Ok((root_hash, notes, total_count)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::drive::shielded::paths::SHIELDED_NOTES_CHUNK_POWER; + use crate::util::test_helpers::setup::setup_drive_with_initial_state_structure; + use platform_version::version::PlatformVersion; + + /// Insert `n` distinct notes into the shielded pool's CommitmentTree. + fn insert_notes(drive: &Drive, n: u64, platform_version: &PlatformVersion) { + for i in 0..n { + let mut cmx = [0u8; 32]; + cmx[..8].copy_from_slice(&i.to_be_bytes()); + let mut nullifier = [0u8; 32]; + nullifier[24..].copy_from_slice(&i.to_be_bytes()); + let nullifier = { + // Ensure nullifiers are also distinct from cmx values. + let mut n = nullifier; + n[0] = 0xCC; + n + }; + + let ops = + Drive::insert_note_op(nullifier, cmx, vec![(i % 256) as u8; 216], platform_version) + .expect("build note op"); + let grove_ops = + crate::fees::op::LowLevelDriveOperation::grovedb_operations_batch_consume(ops); + drive + .grove_apply_batch_with_add_costs( + grove_ops, + false, + None, + &mut vec![], + &platform_version.drive, + ) + .expect("apply note insert"); + } + } + + /// Build the server-side `GetShieldedEncryptedNotes` PathQuery for a + /// chunk `[start_index, start_index + limit)`. Must match + /// `query_shielded_encrypted_notes_v0` byte-for-byte so the proof we + /// produce is the real note-fetch proof shape. + fn notes_fetch_path_query(start_index: u64, limit: u16) -> PathQuery { + let end_index = start_index + limit as u64 - 1; + let mut inner_query = Query::new(); + inner_query.insert_range_inclusive( + start_index.to_be_bytes().to_vec()..=end_index.to_be_bytes().to_vec(), + ); + + PathQuery { + path: shielded_credit_pool_path_vec(), + query: SizedQuery { + query: Query { + items: vec![QueryItem::Key(vec![SHIELDED_NOTES_KEY])], + default_subquery_branch: SubqueryBranch { + subquery_path: None, + subquery: Some(inner_query.into()), + }, + left_to_right: true, + conditional_subquery_branches: None, + add_parent_tree_on_subquery: false, + }, + limit: None, + offset: None, + }, + } + } + + /// THE make-or-break test: prove that `total_count` is extractable from + /// the real `GetShieldedEncryptedNotes` proof. + /// + /// (a) insert N notes, (b) generate a real note-fetch proof via the + /// drive prove path (`grove_get_proved_path_query_v1`, exactly what the + /// drive-abci query handler uses), (c) run the verifier against that + /// proof, and (d) assert the extracted `total_count == N`. + /// + /// If the parent CommitmentTree element were NOT in the note-fetch + /// proof, the inner `verify_subset_query` would fail or return `None` + /// and this would not see `total_count == N`. + #[test] + fn total_count_extractable_from_encrypted_notes_proof() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + + const N: u64 = 5; + insert_notes(&drive, N, platform_version); + + // Request the first chunk. max_elements mirrors the wire cap; we + // only need a limit that covers our N notes. + let mmr_chunk_size: u64 = 1u64 << SHIELDED_NOTES_CHUNK_POWER; + let max_elements = mmr_chunk_size as u32; + let start_index = 0u64; + let limit = max_elements.min(u16::MAX as u32) as u16; + + let path_query = notes_fetch_path_query(start_index, limit); + let proof = drive + .grove_get_proved_path_query_v1(&path_query, &mut vec![], &platform_version.drive) + .expect("should produce note-fetch proof"); + + // verify_subset_of_proof = false: this is a self-contained proof for + // the note-fetch PathQuery (same as the FromProof path). + let (root_hash, notes, total_count) = Drive::verify_shielded_encrypted_notes_v0( + proof.as_slice(), + start_index, + 0, // count = 0 → effective = max_elements + max_elements, + false, + platform_version, + ) + .expect("should verify note-fetch proof and extract total_count"); + + assert!(!root_hash.is_empty(), "root hash should not be empty"); + assert_eq!( + notes.len() as u64, + N, + "should return all {N} inserted notes" + ); + assert_eq!( + total_count, N, + "extracted total_count must equal the number of notes appended on-chain" + ); + } + + /// Same extraction works when the verifier is run in subset mode (the + /// production `FromProof` path passes `verify_subset_of_proof = false`, + /// but the inner count extraction always uses subset semantics — assert + /// both outer modes yield the same `total_count`). + #[test] + fn total_count_stable_across_verify_modes() { + let drive = setup_drive_with_initial_state_structure(None); + let platform_version = PlatformVersion::latest(); + + const N: u64 = 3; + insert_notes(&drive, N, platform_version); + + let mmr_chunk_size: u64 = 1u64 << SHIELDED_NOTES_CHUNK_POWER; + let max_elements = mmr_chunk_size as u32; + let limit = max_elements.min(u16::MAX as u32) as u16; + let path_query = notes_fetch_path_query(0, limit); + let proof = drive + .grove_get_proved_path_query_v1(&path_query, &mut vec![], &platform_version.drive) + .expect("should produce note-fetch proof"); + + let (_, _, tc_full) = Drive::verify_shielded_encrypted_notes_v0( + proof.as_slice(), + 0, + 0, + max_elements, + false, + platform_version, + ) + .expect("verify (non-subset outer)"); + + let (_, _, tc_subset) = Drive::verify_shielded_encrypted_notes_v0( + proof.as_slice(), + 0, + 0, + max_elements, + true, + platform_version, + ) + .expect("verify (subset outer)"); + + assert_eq!(tc_full, N); + assert_eq!(tc_subset, N); } } diff --git a/packages/rs-platform-version/Cargo.toml b/packages/rs-platform-version/Cargo.toml index febc2c1e94e..125441aaaa8 100644 --- a/packages/rs-platform-version/Cargo.toml +++ b/packages/rs-platform-version/Cargo.toml @@ -11,7 +11,7 @@ license = "MIT" thiserror = { version = "2.0.12" } bincode = { version = "=2.0.1" } versioned-feature-core = { git = "https://github.com/dashpay/versioned-feature-core", version = "1.0.0" } -grovedb-version = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9" } +grovedb-version = { git = "https://github.com/dashpay/grovedb", rev = "5eb7a5380a6e974513343352acfd6b30a8c1f87c" } [features] mock-versions = [] diff --git a/packages/rs-platform-wallet-ffi/src/event_handler.rs b/packages/rs-platform-wallet-ffi/src/event_handler.rs index 0bfa06085fa..1213eece587 100644 --- a/packages/rs-platform-wallet-ffi/src/event_handler.rs +++ b/packages/rs-platform-wallet-ffi/src/event_handler.rs @@ -14,6 +14,25 @@ use std::os::raw::{c_char, c_void}; /// /// All callbacks are optional (`Option`) — pass null for events you don't /// care about. The default behavior is to ignore the event. +/// +/// # ABI growth constraint (accepted limitation) +/// +/// New callback slots are appended at the **end** of this struct, which +/// preserves the byte offsets of all pre-existing fields. That keeps the +/// ABI stable for callers that read individual fields, but it does NOT make +/// by-value struct growth safe: `platform_wallet_manager_create` consumes +/// the struct via `std::ptr::read`, which copies `size_of::()` bytes +/// from the caller's pointer. A caller compiled against an older, shorter +/// copy of the generated header allocates a smaller struct, so each appended +/// slot makes `ptr::read` over-read past that allocation. +/// +/// This is accepted for now because the only consumer is the in-tree Swift +/// SDK, which is regenerated (cbindgen) and rebuilt in lockstep with this +/// crate — there is no out-of-tree consumer pinned to a stale header. A +/// proper fix (a leading `size`/`version` discriminator passed to +/// `platform_wallet_manager_create`, or per-callback registration +/// entrypoints) is tracked as a follow-up and is out of scope for the +/// sync-progress work that added these slots. #[repr(C)] pub struct EventHandlerCallbacks { /// Opaque context pointer passed to all callbacks. @@ -47,6 +66,31 @@ pub struct EventHandlerCallbacks { sync_unix_seconds: u64, ), >, + /// Called once per chunk during a shielded sync pass (~every + /// 2048 notes processed). Carries the cumulative count of + /// encrypted notes scanned so far in the current pass plus the + /// latest block height observed. Lets the host render a live + /// progress counter / ProgressView during long cold syncs. + /// Slot is plumbed unconditionally for C-ABI stability; only + /// fires when the `shielded` feature is enabled in the FFI. + pub on_shielded_sync_progress_fn: Option< + unsafe extern "C" fn(context: *mut c_void, cumulative_scanned: u64, block_height: u64), + >, + /// Called once per committed batch during a shielded sync pass as + /// decrypted commitments are appended to the local Orchard tree. + /// This is the "checked / committed-to-tree" signal, distinct from + /// `on_shielded_sync_progress_fn` (which counts *downloaded* + /// notes). `leaves_committed` is the cumulative tree leaf count; + /// `total_target` is the on-chain MMR total leaf count, with + /// `total_target == 0` meaning the total is **indeterminate** (the + /// count RPC was unavailable). Pairs with the download progress + /// callback to drive a dual ProgressView during cold syncs. Slot + /// is plumbed unconditionally for C-ABI stability; only fires when + /// the `shielded` feature is enabled in the FFI. Appended at the + /// end of the struct to preserve existing field offsets. + pub on_shielded_tree_progress_fn: Option< + unsafe extern "C" fn(context: *mut c_void, leaves_committed: u64, total_target: u64), + >, } // SAFETY: The context pointer is managed by the FFI caller who must ensure @@ -207,4 +251,24 @@ impl PlatformEventHandler for FFIEventHandler { ); } } + + #[cfg(feature = "shielded")] + fn on_shielded_sync_progress(&self, cumulative_scanned: u64, block_height: u64) { + let Some(cb) = self.callbacks.on_shielded_sync_progress_fn else { + return; + }; + unsafe { + cb(self.callbacks.context, cumulative_scanned, block_height); + } + } + + #[cfg(feature = "shielded")] + fn on_shielded_tree_progress(&self, leaves_committed: u64, total_target: u64) { + let Some(cb) = self.callbacks.on_shielded_tree_progress_fn else { + return; + }; + unsafe { + cb(self.callbacks.context, leaves_committed, total_target); + } + } } diff --git a/packages/rs-platform-wallet-ffi/src/manager.rs b/packages/rs-platform-wallet-ffi/src/manager.rs index 0f46906fcd1..5930c1c4db6 100644 --- a/packages/rs-platform-wallet-ffi/src/manager.rs +++ b/packages/rs-platform-wallet-ffi/src/manager.rs @@ -352,7 +352,15 @@ pub unsafe extern "C" fn platform_wallet_manager_destroy( handle: Handle, ) -> PlatformWalletFFIResult { if let Some(manager) = PLATFORM_WALLET_MANAGER_STORAGE.remove(handle) { - manager.platform_address_sync().stop(); + // Run the full lifecycle shutdown to completion, not just the + // platform-address sync. Every background task (identity sync, + // shielded sync, the wallet-event adapter) can fire callbacks + // through the host-owned `context` pointer; once `destroy` + // returns the host may free that context, so no task may be + // left alive to fire a callback against freed memory. + // `shutdown()` is idempotent, so this is safe even if the host + // already stopped some sync managers before calling destroy. + runtime().block_on(manager.shutdown()); } PlatformWalletFFIResult::ok() } diff --git a/packages/rs-platform-wallet-ffi/src/shielded_sync.rs b/packages/rs-platform-wallet-ffi/src/shielded_sync.rs index 3f152059c87..2d58d8165f6 100644 --- a/packages/rs-platform-wallet-ffi/src/shielded_sync.rs +++ b/packages/rs-platform-wallet-ffi/src/shielded_sync.rs @@ -406,15 +406,22 @@ pub unsafe extern "C" fn platform_wallet_manager_configure_shielded( /// network-scoped coordinator, and reset the caught-up cooldown /// stamp. /// -/// The single SQLite commitment-tree file stays open — Clear -/// semantics are "wipe my host-side persistence and start -/// re-syncing from index 0 on the shared tree", **not** "blow -/// away the chain-wide cache". The host is responsible for -/// wiping its own per-wallet persistence layer (e.g. SwiftData -/// rows) since Rust can't reach into iOS / Android persistence; -/// after that, the next [`platform_wallet_manager_bind_shielded`] -/// call repopulates the coordinator's registries and the next -/// sync pass re-saves notes via the changeset path. +/// The SQLite commitment-tree file stays on disk but its contents +/// are reset to empty — Clear semantics are "wipe my shielded +/// state and cold-resync from index 0 on the shared tree". The +/// host is responsible for wiping its own per-wallet persistence +/// layer (e.g. SwiftData rows) since Rust can't reach into iOS / +/// Android persistence; after that, the next +/// [`platform_wallet_manager_bind_shielded`] call repopulates the +/// coordinator's registries and the next sync pass re-saves notes +/// via the changeset path. +/// +/// Returns `ErrorWalletOperation` if the Rust-side store reset +/// fails. The host **must** check this before wiping its own +/// persistence: a silent failure would leave the shared tree +/// populated while the host drops its rows, and the next cold +/// resync would gate-skip every re-downloaded position against the +/// stale tree size. /// /// Idempotent: calling Clear when shielded support has never /// been configured (no coordinator installed) is still a @@ -428,13 +435,19 @@ pub unsafe extern "C" fn platform_wallet_manager_shielded_clear( // Single library call: `clear_shielded` quiesces the sync // manager (cancel + drain the in-flight pass, incl. persister // fan-out, so nothing re-persists after Clear) and then clears - // the coordinator registries. Keeping the quiesce+clear - // sequencing in the library (not stitched here) follows the - // FFI's "resolve handle, call one function, marshal result" - // contract. - runtime().block_on(manager.clear_shielded()); + // the coordinator registries + resets the shared store. Keeping + // the quiesce+clear sequencing in the library (not stitched + // here) follows the FFI's "resolve handle, call one function, + // marshal result" contract. + runtime().block_on(manager.clear_shielded()) }); - unwrap_option_or_return!(option); + let result = unwrap_option_or_return!(option); + if let Err(e) = result { + return PlatformWalletFFIResult::err( + PlatformWalletFFIResultCode::ErrorWalletOperation, + format!("clear_shielded failed: {e}"), + ); + } PlatformWalletFFIResult::ok() } diff --git a/packages/rs-platform-wallet/Cargo.toml b/packages/rs-platform-wallet/Cargo.toml index d812c764b9b..b92f03cfa14 100644 --- a/packages/rs-platform-wallet/Cargo.toml +++ b/packages/rs-platform-wallet/Cargo.toml @@ -49,11 +49,48 @@ image = { version = "0.25", default-features = false, features = ["png", "jpeg", zeroize = "1" # Shielded pool (optional, behind `shielded` feature) -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9", optional = true } +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "5eb7a5380a6e974513343352acfd6b30a8c1f87c", optional = true } +# Direct `rusqlite` access so `FileBackedShieldedStore::open_path` can set +# WAL + synchronous=NORMAL pragmas before handing the connection to +# `ClientPersistentCommitmentTree`. Version locked to match the rev grovedb +# is built against (its `Cargo.lock` resolves rusqlite at this major). +rusqlite = { version = "0.38", optional = true } zip32 = { version = "0.2.0", default-features = false, optional = true } +# `StreamExt`/`pin_mut!` to consume the SDK's `sync_shielded_notes_stream` +# in the interleaved fetch/tree-append consumer (shielded sync only). +# Same version as `dash-sdk` so the lockfile resolves a single copy. +futures = { version = "0.3.30", optional = true } + +# Networked, opt-in example binaries. Each one performs real network I/O +# against a live devnet, so they are examples (compiled, never run by +# `cargo test`) rather than `#[ignore]`d tests. They are crate-gated on the +# `shielded` feature (`#![cfg(feature = "shielded")]`), so `required-features` +# lets cargo skip building them entirely when the feature is off — otherwise +# the emptied file would have no `main`. +[[example]] +name = "shielded_sync" +required-features = ["shielded"] + +[[example]] +name = "shielded_sync_paloma" +required-features = ["shielded"] + +[[example]] +name = "shielded_chunk_timing_bench" +required-features = ["shielded"] [dev-dependencies] +# Used by `examples/shielded_chunk_timing_bench.rs` and +# `tests/shielded_decrypt_bench.rs` to assemble per-chunk wire +# fixtures and decode the `ShieldedEncryptedNote` wire type. +drive-proof-verifier = { path = "../rs-drive-proof-verifier" } rand = "0.8" +# Drives the parallel decrypt benchmark in `shielded_decrypt_bench.rs`. +rayon = "1.10" +# Used by `shielded_tree_append_bench.rs` to open SQLite with tuned PRAGMAs +# (WAL vs DELETE, sync mode) — confirms whether journal mode is what's +# making the simulator path so slow. +rusqlite = "0.38" static_assertions = "1.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } # `test-util` brings `tokio::time::pause` / `tokio::test(start_paused = true)` @@ -63,13 +100,17 @@ tokio = { version = "1", features = ["sync", "rt", "time", "macros", "test-util" # Re-enable the SDK with mocks feature for test-only mock builders; # the non-test build keeps the leaner default-feature SDK above. dash-sdk = { path = "../rs-sdk", default-features = false, features = ["dashpay-contract", "dpns-contract", "wallet", "mocks"] } +# Used by `examples/shielded_sync_paloma.rs` to build the SDK against +# a remote devnet that doesn't have Core RPC reachable — supplies +# proof verification via a separate HTTP quorum-list service. +rs-sdk-trusted-context-provider = { path = "../rs-sdk-trusted-context-provider" } [features] default = ["bls", "eddsa"] bls = ["key-wallet/bls", "key-wallet-manager/bls"] eddsa = ["key-wallet/eddsa", "key-wallet-manager/eddsa"] -shielded = ["dep:grovedb-commitment-tree", "dep:zip32", "dash-sdk/shielded", "dpp/shielded-client"] +shielded = ["dep:grovedb-commitment-tree", "dep:rusqlite", "dep:zip32", "dep:futures", "dash-sdk/shielded", "dpp/shielded-client"] # Opt-in serde derives on the changeset types. Activates `key-wallet/serde`, # `key-wallet-manager/serde`, and `dash-sdk/serde`. `dpp` derives serde unconditionally. serde = [ diff --git a/packages/rs-platform-wallet/examples/shielded_chunk_timing_bench.rs b/packages/rs-platform-wallet/examples/shielded_chunk_timing_bench.rs new file mode 100644 index 00000000000..9e58f3faa10 --- /dev/null +++ b/packages/rs-platform-wallet/examples/shielded_chunk_timing_bench.rs @@ -0,0 +1,234 @@ +//! Focused per-chunk timing benchmark against a live SDK_TEST_DATA devnet. +//! +//! Issues N sequential single-chunk `ShieldedEncryptedNotes` fetches against +//! paloma and reports the distribution of per-chunk wall-clock. With the +//! `info!`-level "shielded chunk fetched" telemetry that landed in +//! `rs-sdk/src/platform/fetch.rs::fetch_with_metadata_and_proof`, each call +//! also logs its own `net_ms` (gRPC roundtrip) / `verify_ms` (proof +//! verification) split — so we get both the aggregate distribution and the +//! per-call breakdown without further instrumentation. +//! +//! This is an opt-in example (NOT a test) because it performs real network +//! I/O against a live devnet; cargo examples are compiled but never run by +//! `cargo test`. +//! +//! Run: +//! cargo run -p platform-wallet --release --features shielded \ +//! --example shielded_chunk_timing_bench +//! +//! Env overrides (all optional): +//! - `PALOMA_QUORUM_URL` defaults to http://44.238.203.84:8080 +//! - `PALOMA_DAPI_ADDRESSES` comma-separated https://:1443 list; otherwise +//! 13 paloma defaults are used. +//! - `BENCH_CHUNKS` how many sequential chunks to fetch (default 20) +//! - `BENCH_CHUNK_SIZE` per-chunk size (default 2048; max enforced +//! server-side via the protocol version) +//! - `BENCH_START_INDEX` where to start fetching (default 0) +//! +//! Why sequential (not concurrent): the question this bench answers is +//! "what's the per-chunk cost?", not "what's the network throughput?". 16-way +//! concurrency is already in the production cold-sync code; this bench +//! isolates one chunk at a time so distribution stats reflect true per-call +//! cost (network + server proof gen + client verify) without contention +//! between chunks. + +#![cfg(feature = "shielded")] + +use std::sync::Arc; +use std::time::Instant; + +use dash_sdk::platform::Fetch; +use dash_sdk::sdk::{Address, AddressList}; +use dash_sdk::SdkBuilder; +use dashcore::Network; +use drive_proof_verifier::types::{ShieldedEncryptedNotes, ShieldedEncryptedNotesQuery}; +use rs_sdk_trusted_context_provider::TrustedHttpContextProvider; + +const DEFAULT_QUORUM_URL: &str = "http://44.238.203.84:8080"; +const DEFAULT_DAPI_ADDRESSES: &[&str] = &[ + "https://68.67.122.85:1443", + "https://68.67.122.86:1443", + "https://68.67.122.87:1443", + "https://68.67.122.88:1443", + "https://68.67.122.192:1443", + "https://68.67.122.193:1443", + "https://68.67.122.195:1443", + "https://68.67.122.196:1443", + "https://68.67.122.197:1443", + "https://68.67.122.198:1443", + "https://68.67.122.199:1443", + "https://68.67.122.206:1443", + "https://68.67.122.207:1443", +]; + +fn dapi_addresses() -> AddressList { + let raw = std::env::var("PALOMA_DAPI_ADDRESSES").unwrap_or_default(); + let addrs: Vec
= if raw.trim().is_empty() { + DEFAULT_DAPI_ADDRESSES + .iter() + .filter_map(|s| s.parse().ok()) + .collect() + } else { + raw.split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .filter_map(|s| s.parse().ok()) + .collect() + }; + assert!(!addrs.is_empty(), "no DAPI addresses configured"); + AddressList::from_iter(addrs) +} + +fn quorum_url() -> String { + std::env::var("PALOMA_QUORUM_URL").unwrap_or_else(|_| DEFAULT_QUORUM_URL.to_string()) +} + +fn percentile(sorted: &[u128], pct: f64) -> u128 { + if sorted.is_empty() { + return 0; + } + let idx = ((sorted.len() as f64 - 1.0) * pct / 100.0).round() as usize; + sorted[idx.min(sorted.len() - 1)] +} + +#[tokio::main(flavor = "multi_thread", worker_threads = 4)] +async fn main() { + let _ = tracing_subscriber::FmtSubscriber::builder() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .with_target(true) + .try_init(); + + let chunks: u64 = std::env::var("BENCH_CHUNKS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(20); + let chunk_size: u32 = std::env::var("BENCH_CHUNK_SIZE") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(2048); + let start_index: u64 = std::env::var("BENCH_START_INDEX") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(0); + + let addresses = dapi_addresses(); + let quorum = quorum_url(); + let live_count = addresses.get_live_addresses().len(); + println!("== Shielded chunk timing bench =="); + println!("Quorum URL: {}", quorum); + println!("DAPI nodes: {}", live_count); + println!("Chunks to fetch: {}", chunks); + println!("Chunk size: {}", chunk_size); + println!("Start index: {}", start_index); + + let provider = TrustedHttpContextProvider::new_with_url( + Network::Devnet, + quorum.clone(), + std::num::NonZeroUsize::new(100).unwrap(), + ) + .expect("build TrustedHttpContextProvider"); + + let sdk = SdkBuilder::new(addresses) + .with_network(Network::Devnet) + .with_context_provider(provider) + .build() + .expect("build sdk"); + let sdk = Arc::new(sdk); + + // Warm-up: prefetch_quorums should already have happened in build(); + // do one throwaway fetch to amortize any first-call connection setup. + println!("Warm-up fetch..."); + let _ = ShieldedEncryptedNotes::fetch_with_metadata( + sdk.as_ref(), + ShieldedEncryptedNotesQuery { + start_index, + count: chunk_size, + }, + None, + ) + .await; + + let mut totals_ms: Vec = Vec::with_capacity(chunks as usize); + let mut total_notes: usize = 0; + + // Server requires start_index to be a multiple of the protocol's + // server-side chunk size (2048 today). We step by that constant so + // every request is alignment-valid even when the user asks for a + // smaller `count` — that lets us isolate per-request overhead vs + // bandwidth without tripping the alignment check. + const SERVER_CHUNK_ALIGNMENT: u64 = 2048; + + let bench_start = Instant::now(); + for i in 0..chunks { + let idx = start_index + i * SERVER_CHUNK_ALIGNMENT; + let q = ShieldedEncryptedNotesQuery { + start_index: idx, + count: chunk_size, + }; + let call_start = Instant::now(); + let result = ShieldedEncryptedNotes::fetch_with_metadata(sdk.as_ref(), q, None).await; + let call_ms = call_start.elapsed().as_millis(); + match result { + Ok((Some(ShieldedEncryptedNotes { notes, .. }), _)) => { + total_notes += notes.len(); + println!( + " chunk {:>3} @ idx {:>8} -> {:>6.0} ms ({} notes)", + i, + idx, + call_ms as f64, + notes.len() + ); + } + Ok((None, _)) => { + println!( + " chunk {:>3} @ idx {:>8} -> {:>6.0} ms (empty)", + i, idx, call_ms as f64 + ); + } + Err(e) => { + println!(" chunk {:>3} @ idx {:>8} -> ERROR: {}", i, idx, e); + continue; + } + } + totals_ms.push(call_ms); + } + let bench_elapsed = bench_start.elapsed(); + + if totals_ms.is_empty() { + println!("No chunks succeeded."); + return; + } + + let mut sorted = totals_ms.clone(); + sorted.sort_unstable(); + let sum_ms: u128 = sorted.iter().sum(); + let avg_ms = sum_ms as f64 / sorted.len() as f64; + + println!(); + println!("== Per-chunk distribution (ms, sequential fetches) =="); + println!("Count: {}", sorted.len()); + println!("Min: {:>7.0}", *sorted.first().unwrap() as f64); + println!("p50: {:>7.0}", percentile(&sorted, 50.0) as f64); + println!("p90: {:>7.0}", percentile(&sorted, 90.0) as f64); + println!("p95: {:>7.0}", percentile(&sorted, 95.0) as f64); + println!("p99: {:>7.0}", percentile(&sorted, 99.0) as f64); + println!("Max: {:>7.0}", *sorted.last().unwrap() as f64); + println!("Avg: {:>7.1}", avg_ms); + println!(); + println!( + "Total wall-clock: {:.2} s Notes fetched: {} Rate: {:.0} notes/s (sequential)", + bench_elapsed.as_secs_f64(), + total_notes, + total_notes as f64 / bench_elapsed.as_secs_f64() + ); + println!(); + println!( + "NOTE: Each successful chunk also logs `shielded chunk fetched net_ms=N \ + verify_ms=N total_ms=N address=...` at info level — that's the network/\ + verify split for the same calls. Filter for `shielded chunk` in the bench \ + output above to read them in order." + ); +} diff --git a/packages/rs-platform-wallet/examples/shielded_sync.rs b/packages/rs-platform-wallet/examples/shielded_sync.rs new file mode 100644 index 00000000000..154edeff6bd --- /dev/null +++ b/packages/rs-platform-wallet/examples/shielded_sync.rs @@ -0,0 +1,345 @@ +//! Functional example for the SDK_TEST_DATA seeded shielded pool. +//! +//! Drives the **full PlatformWalletManager flow** end-to-end: create wallet +//! → bind shielded → trigger a sync pass via the network coordinator → check +//! the recovered balance. Goes through the same APIs production wallets use, +//! so any breakage in `bind_shielded`, `NetworkShieldedCoordinator::sync`, the +//! gRPC layer, or proof verification surfaces here. +//! +//! This is an opt-in example (NOT a test) because it performs real network +//! I/O against a running devnet; cargo examples are compiled but never run by +//! `cargo test`. +//! +//! The in-process unit tests in `rs-drive-abci/.../create_genesis_state/test/ +//! shielded.rs` prove crypto + drive integration are correct. This example +//! closes the remaining gap by running a real wallet against a real chain. +//! +//! # Expected chain config +//! +//! The seed config is hardcoded on the chain side (see +//! `ShieldedSeedConfig::sdk_test_data` in rs-drive-abci): +//! `total_notes = 5_000, owned_count = 8 (split 4/4), owned_value = 100_000, +//! rng_seed = 0xDEAD_BEEF`. Each wallet's expected balance after sync is +//! `4 × 100_000 = 400_000`. +//! +//! # Requirements +//! +//! 1. A running devnet whose genesis was created with SDK test data. The +//! `local` dashmate config has `buildArgs.SDK_TEST_DATA = "true"` set +//! automatically by `yarn setup`, so: +//! ```bash +//! yarn reset && yarn start +//! ``` +//! +//! 2. `DASH_SDK_CORE_PASSWORD` set to the dashmate Core RPC password +//! (find it in `~/.dashmate/config.json` under +//! `local_1.core.rpc.users.dashmate.password`). +//! +//! 3. Optional `PLATFORM_HOST` / `PLATFORM_PORT` to override the default +//! devnet endpoint (`127.0.0.1:2443`). +//! +//! # Running +//! +//! ```bash +//! # Wallet A (default): +//! DASH_SDK_CORE_PASSWORD='' cargo run -p platform-wallet \ +//! --example shielded_sync --features shielded +//! +//! # Wallet B (cross-wallet privacy check): +//! SHIELDED_SYNC_WALLET=B DASH_SDK_CORE_PASSWORD='' \ +//! cargo run -p platform-wallet --example shielded_sync --features shielded +//! ``` + +#![cfg(feature = "shielded")] + +use std::sync::Arc; + +use dash_sdk::sdk::{Address, AddressList}; +use dash_sdk::SdkBuilder; +use dashcore::Network; +use key_wallet::wallet::initialization::WalletAccountCreationOptions; +use platform_wallet::changeset::{ + ClientStartState, PlatformWalletChangeSet, PlatformWalletPersistence, +}; +use platform_wallet::events::{EventHandler, PlatformEventHandler}; +use platform_wallet::wallet::platform_wallet::WalletId; +use platform_wallet::PlatformWalletManager; + +/// Wallet A seed — **must stay byte-identical** to +/// `packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/shielded_test_wallets.rs::SEED_A`. +/// +/// Both sides derive the IVK via the same ZIP-32 path +/// (`SpendingKey::from_zip32_seed(seed, coin_type=1, account=0)`), so the +/// recipient address the chain encrypts to is byte-identical to the address +/// the wallet's IVK trial-decrypts under. If the chain-side switches +/// derivation, this example fails with "decrypted 0 notes". +const SEED_A: [u8; 32] = [0x73; 32]; + +/// Wallet B seed — see [`SEED_A`]. +const SEED_B: [u8; 32] = [0x74; 32]; + +/// Hardcoded mirror of `ShieldedSeedConfig::sdk_test_data` on the chain side +/// (rs-drive-abci). If the chain-side constant changes, update both — there's +/// no shared crate to import from (rs-drive-abci is downstream of this crate). +const COUNT_A: u32 = 4; +const COUNT_B: u32 = 4; +const OWNED_VALUE: u64 = 100_000; + +const EXPECTED_BALANCE_A: u64 = COUNT_A as u64 * OWNED_VALUE; // 400_000 +const EXPECTED_BALANCE_B: u64 = COUNT_B as u64 * OWNED_VALUE; // 400_000 + +#[derive(Clone, Copy, Debug)] +enum WalletIndex { + A, + B, +} + +impl WalletIndex { + fn seed(self) -> [u8; 32] { + match self { + WalletIndex::A => SEED_A, + WalletIndex::B => SEED_B, + } + } +} + +/// In-memory no-op persister. Real wallets persist; for this example we only +/// care that a single sync pass recovers the right balance. +struct NoopPersister; +impl PlatformWalletPersistence for NoopPersister { + fn store( + &self, + _wallet_id: WalletId, + _changeset: PlatformWalletChangeSet, + ) -> Result<(), platform_wallet::changeset::PersistenceError> { + Ok(()) + } + + fn load(&self) -> Result { + Ok(ClientStartState::default()) + } + + fn flush( + &self, + _wallet_id: WalletId, + ) -> Result<(), platform_wallet::changeset::PersistenceError> { + Ok(()) + } +} + +struct NoopEventHandler; +impl EventHandler for NoopEventHandler {} +impl PlatformEventHandler for NoopEventHandler {} + +/// Drive the full PlatformWalletManager flow for the given wallet and assert +/// the recovered balance equals what the chain's seed config produces. +async fn run_wallet_balance_test(wallet: WalletIndex) { + let _ = tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .with_target(true) + .try_init(); + + let (expected_count, expected_balance) = match wallet { + WalletIndex::A => (COUNT_A, EXPECTED_BALANCE_A), + WalletIndex::B => (COUNT_B, EXPECTED_BALANCE_B), + }; + eprintln!( + "{:?}: expecting {} notes summing to {}", + wallet, expected_count, expected_balance, + ); + + // --- 1. Build SDK pointing at the local devnet --- + // Dashmate's local gateway issues SHA-1-signed certs that modern rustls + // rejects, so the conventional rs-sdk test pattern is to talk HTTP + // (gateway accepts both on the same port). Default `PLATFORM_SSL=false` + // matches `packages/rs-sdk/tests/.env.example`. + let host = std::env::var("PLATFORM_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()); + let port: u16 = std::env::var("PLATFORM_PORT") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(2443); + let use_ssl = std::env::var("PLATFORM_SSL") + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or(false); + let scheme = if use_ssl { "https" } else { "http" }; + let address: Address = format!("{}://{}:{}", scheme, host, port) + .parse() + .expect("parse devnet address"); + let addresses = AddressList::from_iter([address]); + + // Core RPC credentials — the SDK needs these to fetch quorum public keys + // for proof verification. With `with_core(...)` and no explicit + // `with_context_provider`, the SDK auto-installs `GrpcContextProvider` + // which uses Core RPC under the hood. Defaults assume dashmate's + // local_1/seed node; override via `DASH_SDK_CORE_*` env vars (same names + // the rs-sdk fetch tests use). + let core_host = std::env::var("DASH_SDK_CORE_HOST").unwrap_or_else(|_| host.clone()); + let core_port: u16 = std::env::var("DASH_SDK_CORE_PORT") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(20002); + let core_user = std::env::var("DASH_SDK_CORE_USER").unwrap_or_else(|_| "dashmate".to_string()); + let core_password = std::env::var("DASH_SDK_CORE_PASSWORD").unwrap_or_default(); + + let network = Network::Regtest; + let mut builder = SdkBuilder::new(addresses).with_network(network).with_core( + &core_host, + core_port, + &core_user, + &core_password, + ); + + // If the operator explicitly opted into SSL, load dashmate's CA cert + // (overridable via `DASHMATE_CA_CERT`). + if use_ssl { + let ca_cert_path = std::env::var("DASHMATE_CA_CERT").unwrap_or_else(|_| { + let home = std::env::var("HOME").unwrap_or_else(|_| "/".to_string()); + format!("{}/.dashmate/local_1/platform/gateway/ssl/bundle.crt", home) + }); + if std::path::Path::new(&ca_cert_path).exists() { + eprintln!("loading CA cert from {}", ca_cert_path); + builder = builder + .with_ca_certificate_file(&ca_cert_path) + .expect("load CA cert file"); + } + } + eprintln!( + "connecting to platform {}://{}:{}, core rpc {}:{}@{}", + scheme, host, port, core_user, core_port, core_host + ); + + let sdk = builder.build().expect("build sdk"); + let sdk = Arc::new(sdk); + + // --- 2. Build the manager --- + // `PlatformWalletManager::new` is generic over `P: PlatformWalletPersistence`, + // so the persister must stay concrete (an `Arc` would erase the + // type param and break inference downstream). + let persister = Arc::new(NoopPersister); + let event_handler: Arc = Arc::new(NoopEventHandler); + let manager = Arc::new(PlatformWalletManager::new( + Arc::clone(&sdk), + persister, + event_handler, + )); + + // --- 3. Configure shielded support (creates the SQLite store) --- + let shielded_db_dir = std::env::temp_dir().join(format!( + "platform-wallet-shielded-test-{:?}-{}", + wallet, + std::process::id() + )); + std::fs::create_dir_all(&shielded_db_dir).expect("mkdir shielded db dir"); + let shielded_db_path = shielded_db_dir.join("shielded.db"); + + manager + .configure_shielded(&shielded_db_path) + .await + .expect("configure_shielded"); + + // --- 4. Create a platform wallet. The transparent-layer seed is a + // BIP-39-style 64-byte seed and is immaterial for this example + // (we never spend or query transparent state); we duplicate + // the 32-byte shielded seed into a deterministic 64-byte + // pattern so the wallet ID is reproducible per `wallet`. --- + let shielded_seed = wallet.seed(); + let mut transparent_seed = [0u8; 64]; + transparent_seed[..32].copy_from_slice(&shielded_seed); + transparent_seed[32..].copy_from_slice(&shielded_seed); + let platform_wallet = manager + .create_wallet_from_seed_bytes( + network, + transparent_seed, + WalletAccountCreationOptions::Default, + None, + ) + .await + .expect("create_wallet_from_seed_bytes"); + + eprintln!( + "{:?}: created platform wallet id = {}", + wallet, + hex::encode(platform_wallet.wallet_id()) + ); + + // --- 5. Bind the shielded sub-wallet with the SAME seed bytes the + // chain-side seeder uses, deriving via ZIP-32 (account 0) --- + let coordinator = manager + .shielded_coordinator() + .await + .expect("shielded_coordinator must exist after configure_shielded"); + platform_wallet + .bind_shielded(&shielded_seed, &[0u32], &coordinator) + .await + .expect("bind_shielded"); + + // --- 6. Run a single sync pass through the coordinator. `force = true` + // skips the cooldown gate so the example runs immediately after + // the chain comes up. --- + let summary = coordinator.sync(true).await; + eprintln!("{:?}: sync summary: {:?}", wallet, summary); + + // --- 7. Read the wallet's shielded balance per ZIP-32 account. We bound + // account 0 only, so we expect exactly one entry. --- + let balances = platform_wallet + .shielded_balances(&coordinator) + .await + .expect("shielded_balances"); + let total_balance: u64 = balances.values().sum(); + eprintln!( + "{:?}: per-account balances = {:?} (total {})", + wallet, balances, total_balance + ); + + assert_eq!( + total_balance, expected_balance, + "{:?}: balance mismatch (expected {} = {} × {}, got {})", + wallet, expected_balance, expected_count, OWNED_VALUE, total_balance, + ); + + // Best-effort cleanup of the temp SQLite dir. + let _ = std::fs::remove_dir_all(&shielded_db_dir); +} + +/// Sync a wallet against the seeded pool and verify balance = +/// `count × owned_value`. +/// +/// Defaults to wallet A; set `SHIELDED_SYNC_WALLET=B` to run wallet B +/// instead. Running wallet B also pins cross-wallet privacy at the network +/// layer — if A's IVK leaked over the wire and B picked up A's notes, B's +/// balance would exceed `count_b × value`. +#[tokio::main(flavor = "multi_thread", worker_threads = 1)] +async fn main() { + let wallet = match std::env::var("SHIELDED_SYNC_WALLET") + .unwrap_or_default() + .trim() + .to_ascii_uppercase() + .as_str() + { + "" | "A" => WalletIndex::A, + "B" => WalletIndex::B, + other => panic!("invalid SHIELDED_SYNC_WALLET `{other}`; expected A or B"), + }; + + run_wallet_balance_test(wallet).await; +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Keep the hardcoded expectations coherent with the seed constants + /// so `cargo test` (not just a manual example run) catches drift if + /// someone bumps `COUNT_*` / `OWNED_VALUE` without updating the + /// expected balances, or accidentally aliases the two seeds. + #[test] + fn expected_balances_match_seed_constants() { + assert_eq!(EXPECTED_BALANCE_A, COUNT_A as u64 * OWNED_VALUE); + assert_eq!(EXPECTED_BALANCE_B, COUNT_B as u64 * OWNED_VALUE); + assert_ne!(SEED_A, SEED_B); + } +} diff --git a/packages/rs-platform-wallet/examples/shielded_sync_paloma.rs b/packages/rs-platform-wallet/examples/shielded_sync_paloma.rs new file mode 100644 index 00000000000..f22de1e5e57 --- /dev/null +++ b/packages/rs-platform-wallet/examples/shielded_sync_paloma.rs @@ -0,0 +1,277 @@ +//! Remote-devnet variant of the `shielded_sync` example. Binds wallet A +//! (raw ZIP-32 seed `[0x73; 32]`) against a live devnet that's been +//! deployed from a `SDK_TEST_DATA=true` image, drives one shielded sync +//! pass, and asserts the recovered balance. +//! +//! This is an opt-in example (NOT a test) because it performs real network +//! I/O against a live devnet; cargo examples are compiled but never run by +//! `cargo test`. +//! +//! Diverges from the `shielded_sync` example in two ways: +//! +//! 1. **No Core RPC.** The remote devnet's Core RPC ports are +//! firewalled, so we can't use `SdkBuilder::with_core(...)`. Instead +//! we install a [`TrustedHttpContextProvider`] that pulls quorum +//! metadata from a separate HTTP service. Same approach the iOS +//! SwiftExampleApp uses for Devnet (see `SDK.swift::platformQuorumURL`). +//! +//! 2. **Multi-node DAPI fan-out.** Lists every active masternode's +//! Platform HTTP port. `AddressList` picks randomly per request +//! (`rs-dapi-client/src/address_list.rs:213`), so distributing the +//! gRPC traffic across the full masternode set avoids funneling +//! every encrypted-note stream through one gateway. +//! +//! # Hardcoded for paloma +//! +//! Both the quorum URL and the DAPI address list default to paloma's +//! deployment (May 2026). Override via env vars listed below. +//! +//! # Requirements +//! +//! - Paloma devnet (or another devnet built from `SDK_TEST_DATA=true`) +//! reachable from this host. Verify with: +//! ```bash +//! curl http://44.238.203.84:8080/masternodes | jq '.data | length' +//! ``` +//! +//! # Running +//! +//! ```bash +//! cargo run -p platform-wallet --example shielded_sync_paloma --features shielded +//! ``` +//! +//! Env overrides (all optional): +//! - `PALOMA_QUORUM_URL` — defaults to `http://44.238.203.84:8080` +//! - `PALOMA_DAPI_ADDRESSES` — comma-separated `https://:1443` list +//! +//! # What this proves +//! +//! Green: ZIP-32 derivation + bind + decryption + persistence work +//! end-to-end against paloma. If the iOS SwiftExampleApp shows balance +//! = 0 against the same devnet, the bug is iOS-side (persister +//! callback, SwiftData write, UI aggregation). +//! +//! Red on "decrypted 0 notes": paloma was built without +//! `SDK_TEST_DATA=true`, or from a commit predating the snapshot +//! machinery — its 1M notes are filler-only. +//! +//! Red on proof / network errors: quorum URL or DAPI addresses are +//! wrong or unreachable. + +#![cfg(feature = "shielded")] + +use std::sync::Arc; + +use dash_sdk::sdk::{Address, AddressList}; +use dash_sdk::SdkBuilder; +use dashcore::Network; +use key_wallet::wallet::initialization::WalletAccountCreationOptions; +use platform_wallet::changeset::{ + ClientStartState, PlatformWalletChangeSet, PlatformWalletPersistence, +}; +use platform_wallet::events::{EventHandler, PlatformEventHandler}; +use platform_wallet::wallet::platform_wallet::WalletId; +use platform_wallet::PlatformWalletManager; +use rs_sdk_trusted_context_provider::TrustedHttpContextProvider; + +/// Wallet A seed — must stay byte-identical to +/// `packages/rs-drive-abci/.../shielded_test_wallets.rs::SEED_A`. +const SEED_A: [u8; 32] = [0x73; 32]; + +/// Per-wallet bake config (mirror of `ShieldedSeedConfig::sdk_test_data`). +const COUNT_A: u32 = 4; +const OWNED_VALUE: u64 = 100_000; +const EXPECTED_BALANCE_A: u64 = COUNT_A as u64 * OWNED_VALUE; // 400_000 + +/// Default quorum-list service URL for paloma. Override with +/// `PALOMA_QUORUM_URL`. +const DEFAULT_QUORUM_URL: &str = "http://44.238.203.84:8080"; + +/// All 13 active paloma masternodes' Platform HTTP gateways, as of +/// May 2026 (snapshot from +/// `curl http://44.238.203.84:8080/masternodes | jq -r '.data[].address'` +/// with the Platform HTTP port `1443`). Override with +/// `PALOMA_DAPI_ADDRESSES` if the deployment changes. +const DEFAULT_DAPI_ADDRESSES: &[&str] = &[ + "https://68.67.122.85:1443", + "https://68.67.122.86:1443", + "https://68.67.122.87:1443", + "https://68.67.122.88:1443", + "https://68.67.122.192:1443", + "https://68.67.122.193:1443", + "https://68.67.122.195:1443", + "https://68.67.122.196:1443", + "https://68.67.122.197:1443", + "https://68.67.122.198:1443", + "https://68.67.122.199:1443", + "https://68.67.122.206:1443", + "https://68.67.122.207:1443", +]; + +/// In-memory no-op persister — matches the same in-test stub from +/// the `shielded_sync` example. We only need a single sync pass to compute +/// the balance; persisted state across runs is irrelevant. +struct NoopPersister; +impl PlatformWalletPersistence for NoopPersister { + fn store( + &self, + _wallet_id: WalletId, + _changeset: PlatformWalletChangeSet, + ) -> Result<(), platform_wallet::changeset::PersistenceError> { + Ok(()) + } + + fn load(&self) -> Result { + Ok(ClientStartState::default()) + } + + fn flush( + &self, + _wallet_id: WalletId, + ) -> Result<(), platform_wallet::changeset::PersistenceError> { + Ok(()) + } +} + +struct NoopEventHandler; +impl EventHandler for NoopEventHandler {} +impl PlatformEventHandler for NoopEventHandler {} + +/// Parse `PALOMA_DAPI_ADDRESSES` (comma-separated) or fall back to the +/// hardcoded list. Trims whitespace per entry, skips empties. +fn dapi_addresses() -> AddressList { + let raw = std::env::var("PALOMA_DAPI_ADDRESSES").unwrap_or_default(); + let addrs: Vec
= if raw.trim().is_empty() { + DEFAULT_DAPI_ADDRESSES + .iter() + .filter_map(|s| s.parse().ok()) + .collect() + } else { + raw.split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .filter_map(|s| s.parse().ok()) + .collect() + }; + assert!( + !addrs.is_empty(), + "no DAPI addresses configured (PALOMA_DAPI_ADDRESSES empty and default list parse failed?)" + ); + AddressList::from_iter(addrs) +} + +fn quorum_url() -> String { + std::env::var("PALOMA_QUORUM_URL").unwrap_or_else(|_| DEFAULT_QUORUM_URL.to_string()) +} + +#[tokio::main(flavor = "multi_thread", worker_threads = 1)] +async fn main() { + let _ = tracing_subscriber::FmtSubscriber::builder() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), + ) + .with_target(true) + .try_init(); + + let addresses = dapi_addresses(); + let quorum = quorum_url(); + let live_count = addresses.get_live_addresses().len(); + eprintln!( + "paloma config: quorum_url={} dapi_node_count={} expected_balance={}", + quorum, live_count, EXPECTED_BALANCE_A, + ); + + // --- 1. Build SDK pointing at all paloma masternodes, with the + // trusted context provider handling proof verification via + // the HTTP quorum-list service. --- + let provider = TrustedHttpContextProvider::new_with_url( + Network::Devnet, + quorum.clone(), + std::num::NonZeroUsize::new(100).unwrap(), + ) + .expect("build TrustedHttpContextProvider"); + + let sdk = SdkBuilder::new(addresses) + .with_network(Network::Devnet) + .with_context_provider(provider) + .build() + .expect("build sdk"); + let sdk = Arc::new(sdk); + + // --- 2. Manager + shielded coordinator with an ephemeral SQLite DB. --- + let persister = Arc::new(NoopPersister); + let event_handler: Arc = Arc::new(NoopEventHandler); + let manager = Arc::new(PlatformWalletManager::new( + Arc::clone(&sdk), + persister, + event_handler, + )); + + let shielded_db_dir = std::env::temp_dir().join(format!( + "platform-wallet-shielded-paloma-test-{}", + std::process::id() + )); + std::fs::create_dir_all(&shielded_db_dir).expect("mkdir shielded db dir"); + let shielded_db_path = shielded_db_dir.join("shielded.db"); + + manager + .configure_shielded(&shielded_db_path) + .await + .expect("configure_shielded"); + + // --- 3. Create the platform wallet. Transparent seed is immaterial + // for this test; reuse the 32-byte shielded seed twice. --- + let mut transparent_seed = [0u8; 64]; + transparent_seed[..32].copy_from_slice(&SEED_A); + transparent_seed[32..].copy_from_slice(&SEED_A); + let platform_wallet = manager + .create_wallet_from_seed_bytes( + Network::Devnet, + transparent_seed, + WalletAccountCreationOptions::Default, + // birth_height_override: skip SPV-tip lookup (no SPV running here) + Some(0), + ) + .await + .expect("create_wallet_from_seed_bytes"); + + eprintln!( + "wallet_a created: wallet_id = {}", + hex::encode(platform_wallet.wallet_id()) + ); + + // --- 4. Bind shielded account 0 with the raw seed (mirrors the + // iOS `bindShieldedRawSeed` path; same FFI shape). --- + let coordinator = manager + .shielded_coordinator() + .await + .expect("shielded_coordinator must exist after configure_shielded"); + platform_wallet + .bind_shielded(&SEED_A, &[0u32], &coordinator) + .await + .expect("bind_shielded"); + + // --- 5. Run one sync pass with cooldown bypassed. --- + let summary = coordinator.sync(true).await; + eprintln!("sync summary: {:?}", summary); + + // --- 6. Read the per-account balance + assert. --- + let balances = platform_wallet + .shielded_balances(&coordinator) + .await + .expect("shielded_balances"); + let total_balance: u64 = balances.values().sum(); + eprintln!( + "paloma wallet A balances = {:?} (total {})", + balances, total_balance, + ); + + assert_eq!( + total_balance, EXPECTED_BALANCE_A, + "wallet A balance mismatch on paloma (expected {} = {} × {}, got {})", + EXPECTED_BALANCE_A, COUNT_A, OWNED_VALUE, total_balance, + ); + + let _ = std::fs::remove_dir_all(&shielded_db_dir); +} diff --git a/packages/rs-platform-wallet/src/events.rs b/packages/rs-platform-wallet/src/events.rs index e73ed5eb235..9ac256e8730 100644 --- a/packages/rs-platform-wallet/src/events.rs +++ b/packages/rs-platform-wallet/src/events.rs @@ -43,6 +43,53 @@ pub trait PlatformEventHandler: EventHandler { /// [`ShieldedSyncManager`]: crate::manager::shielded_sync::ShieldedSyncManager #[cfg(feature = "shielded")] fn on_shielded_sync_completed(&self, _summary: &ShieldedSyncPassSummary) {} + + /// Fired periodically during a shielded sync pass — once per + /// completed chunk inside `sync_shielded_notes`. Carries the + /// cumulative count of encrypted notes scanned so far in the + /// current pass and the latest block height observed. + /// + /// Network-scoped, not per-wallet: a single sync pass covers + /// every IVK on the coordinator, so the "wallet that's + /// progressing" isn't a meaningful concept at this granularity. + /// Clients that bind a single wallet at a time can scope + /// per-wallet from context. + /// + /// Lets clients render a live progress counter / `ProgressView` + /// during long initial syncs (a cold sync of a 1M-note pool can + /// take 20+ min in a single `sync_shielded_notes` call; without + /// this event there's no signal between start and end). + /// + /// Default impl is a no-op. + #[cfg(feature = "shielded")] + fn on_shielded_sync_progress(&self, _cumulative_scanned: u64, _block_height: u64) {} + + /// Fired periodically during a shielded sync pass — once per + /// committed batch as decrypted commitments are appended to the + /// local Orchard commitment tree. This is the "checked / + /// committed-to-tree" signal, distinct from + /// [`on_shielded_sync_progress`](Self::on_shielded_sync_progress) + /// (which counts *downloaded* notes): a note is only "checked" + /// once its commitment has been appended to the tree. + /// + /// `leaves_committed` is the cumulative tree leaf count (starts at + /// the pre-sync tree size and grows as commitments are appended). + /// `total_target` is the on-chain MMR total leaf count, fetched + /// once at pass start; `total_target == 0` means the count RPC + /// was unavailable, so the progress is **indeterminate** and the + /// host should render a spinner rather than a determinate bar. + /// + /// Network-scoped, not per-wallet: a single sync pass covers + /// every IVK on the coordinator, so the "wallet that's + /// progressing" isn't a meaningful concept at this granularity. + /// + /// Pairs with `on_shielded_sync_progress` to drive a dual + /// ProgressView ("downloaded" vs "checked") during long cold + /// syncs. + /// + /// Default impl is a no-op. + #[cfg(feature = "shielded")] + fn on_shielded_tree_progress(&self, _leaves_committed: u64, _total_target: u64) {} } /// Dispatches events to all registered [`PlatformEventHandler`]s. @@ -94,6 +141,35 @@ impl PlatformEventManager { h.on_shielded_sync_completed(summary); } } + + /// Dispatch a shielded sync progress event to every handler. + /// + /// Called from inside `sync_shielded_notes`'s chunk loop, once + /// per chunk (~every 2048 notes processed). Cheap-but-frequent + /// path during a cold sync. + #[cfg(feature = "shielded")] + pub fn on_shielded_sync_progress(&self, cumulative_scanned: u64, block_height: u64) { + let handlers = self.handlers.load(); + for h in handlers.iter() { + h.on_shielded_sync_progress(cumulative_scanned, block_height); + } + } + + /// Dispatch a shielded tree-progress ("checked / committed-to-tree") + /// event to every handler. + /// + /// Called from inside `sync_notes_across`'s batch loop, once per + /// committed batch as commitments are appended to the local + /// Orchard tree. `total_target == 0` signals an indeterminate + /// total (the on-chain MMR leaf count was unavailable). Cheap-but- + /// frequent path during a cold sync. + #[cfg(feature = "shielded")] + pub fn on_shielded_tree_progress(&self, leaves_committed: u64, total_target: u64) { + let handlers = self.handlers.load(); + for h in handlers.iter() { + h.on_shielded_tree_progress(leaves_committed, total_target); + } + } } impl EventHandler for PlatformEventManager { diff --git a/packages/rs-platform-wallet/src/manager/identity_sync.rs b/packages/rs-platform-wallet/src/manager/identity_sync.rs index 7023190d91f..8730398f978 100644 --- a/packages/rs-platform-wallet/src/manager/identity_sync.rs +++ b/packages/rs-platform-wallet/src/manager/identity_sync.rs @@ -166,6 +166,13 @@ where background_generation: AtomicU64, interval_secs: AtomicU64, is_syncing: AtomicBool, + /// Set by [`quiesce`](Self::quiesce) to gate new passes while it + /// drains an in-flight one. `sync_now` bails (after taking the + /// `is_syncing` slot) when this is set, so once `quiesce` observes + /// `is_syncing == false` no further pass can start — giving shutdown + /// a real "no more host-visible persister stores" barrier that + /// cancel-only [`stop`](Self::stop) does not provide. + quiescing: AtomicBool, /// Unix seconds of the last completed pass across all identities. /// `0` = never. Identity-level timestamps live on the per-identity /// rows in [`IdentitySyncManager::state`]. @@ -200,6 +207,7 @@ where background_generation: AtomicU64::new(0), interval_secs: AtomicU64::new(DEFAULT_SYNC_INTERVAL_SECS), is_syncing: AtomicBool::new(false), + quiescing: AtomicBool::new(false), last_sync_unix: AtomicU64::new(0), state: RwLock::new(BTreeMap::new()), } @@ -429,6 +437,13 @@ where } /// Stop the background sync loop. No-op if not running. + /// + /// **Cancel-only**: requests cancellation and returns immediately. A + /// pass already inside `sync_now` keeps running to completion, + /// including its `persister.store(...)` fan-out. For a real "nothing + /// is running and nothing more will be persisted" barrier — required + /// by manager shutdown so the host can free the persister context — + /// use [`quiesce`](Self::quiesce). pub fn stop(&self) { if let Some(token) = self .background_cancel @@ -440,6 +455,33 @@ where } } + /// Cancel the background loop **and wait for any in-flight sync pass + /// to fully drain** before returning — a real quiescence barrier, + /// unlike cancel-only [`stop`](Self::stop). + /// + /// After this returns, no sync pass is running and none can start + /// until the next [`start`](Self::start) / `sync_now`, so a caller + /// that immediately tears the manager down (and frees the host-owned + /// persister context the FFI handed to us) cannot be raced by a pass + /// that calls `persister.store(...)` through a now-dangling pointer. + /// + /// Mechanism: set the `quiescing` gate so any pass that hasn't yet + /// taken the `is_syncing` slot bails, cancel the loop, then wait for + /// `is_syncing` to clear. `is_syncing` is held for the whole pass + /// including the persister fan-out (`sync_now` clears it only after + /// every `sync_identity` / `apply_fresh_balances` store completes), + /// so its falling edge (with the gate up) is a sound "fully drained" + /// signal. The gate is reopened before returning so a later + /// start/sync works normally. + pub async fn quiesce(&self) { + self.quiescing.store(true, Ordering::Release); + self.stop(); + while self.is_syncing.load(Ordering::Acquire) { + tokio::time::sleep(Duration::from_millis(20)).await; + } + self.quiescing.store(false, Ordering::Release); + } + /// Run one sync pass across every registered identity. /// /// If a pass is already in flight, returns immediately without @@ -459,6 +501,15 @@ where return; } + // A `quiesce()` may have raised the gate between our CAS and + // here; if so, release the slot and bail without running a pass + // so the drain can complete and shutdown gets a true barrier + // (no further `persister.store(...)` after quiesce returns). + if self.quiescing.load(Ordering::Acquire) { + self.is_syncing.store(false, Ordering::Release); + return; + } + // Snapshot the per-identity watch list under a short read // lock and release it before any network call. We keep // `Vec` in token-id order so each batch chunk is @@ -829,6 +880,85 @@ mod tests { assert_eq!(mgr.interval(), Duration::from_secs(120)); } + /// `quiesce()` must not return while a pass is in flight, and must + /// return promptly once the pass drains. + /// + /// Drives the real `is_syncing` lifecycle: a background task takes + /// the slot via the same `compare_exchange` the real `sync_now` + /// uses, holds it across a sleep (standing in for the pass body + + /// persister fan-out, which `sync_now` keeps the flag set across), + /// then clears it. We assert `quiesce()` is still pending while the + /// flag is held and completes after it falls — i.e. the falling edge + /// of `is_syncing` is what unblocks the barrier. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn quiesce_blocks_until_in_flight_pass_drains() { + let mgr = make_manager(); + + // Stand in for an in-flight `sync_now`: take the `is_syncing` + // slot exactly as the real pass does, hold it, then release. + let holder = Arc::clone(&mgr); + let pass = tokio::spawn(async move { + assert!( + holder + .is_syncing + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok(), + "test should own the is_syncing slot" + ); + tokio::time::sleep(Duration::from_millis(200)).await; + holder.is_syncing.store(false, Ordering::Release); + }); + + // Give the holder task a chance to take the slot before we + // start draining. + while !mgr.is_syncing() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + + let quiesce_fut = mgr.quiesce(); + tokio::pin!(quiesce_fut); + + // While the pass holds the flag, quiesce must stay pending. + tokio::select! { + _ = &mut quiesce_fut => panic!("quiesce returned while a pass was in flight"), + _ = tokio::time::sleep(Duration::from_millis(50)) => {} + } + assert!(mgr.is_syncing(), "pass should still be in flight"); + + // Once the pass drains, quiesce must return (well within a + // generous bound — it polls every 20ms). + tokio::time::timeout(Duration::from_secs(2), &mut quiesce_fut) + .await + .expect("quiesce did not return after the pass drained"); + + // The gate is reopened before quiesce returns. + assert!(!mgr.quiescing.load(Ordering::Acquire)); + assert!(!mgr.is_syncing()); + pass.await.unwrap(); + } + + /// A `sync_now()` invoked while `quiescing` is set must bail without + /// running the pass — in particular, without calling + /// `persister.store(...)`. This is the gate that prevents a pass + /// from slipping in between `quiesce`'s `stop()` and its drain. + #[tokio::test] + async fn sync_now_bails_when_quiescing() { + let (mgr, persister) = make_recording_manager(); + let id_a = Identifier::from([1u8; 32]); + let token_x = Identifier::from([10u8; 32]); + mgr.register_identity(id_a, [token_x]).await; + + // Raise the gate as `quiesce()` would. + mgr.quiescing.store(true, Ordering::Release); + + mgr.sync_now().await; + + // No persister store happened, and the slot was released so a + // later (post-quiesce) pass can still run. + assert_eq!(persister.stores.load(AtomicOrdering::SeqCst), 0); + assert!(!mgr.is_syncing()); + } + /// Round-trip: register → read → update_watched_tokens → read. /// `update_watched_tokens` preserves the rows for tokens still in /// the new set, drops removed ones, and inserts placeholders for diff --git a/packages/rs-platform-wallet/src/manager/mod.rs b/packages/rs-platform-wallet/src/manager/mod.rs index 78fc7db3c55..b6223b7023c 100644 --- a/packages/rs-platform-wallet/src/manager/mod.rs +++ b/packages/rs-platform-wallet/src/manager/mod.rs @@ -71,6 +71,13 @@ pub struct PlatformWalletManager { #[cfg(feature = "shielded")] pub(super) shielded_coordinator: Arc>>>, + /// Shared `PlatformEventManager` — held on the manager so + /// `configure_shielded` can install a per-chunk progress handler + /// onto the freshly-created `NetworkShieldedCoordinator` that + /// forwards into `on_shielded_sync_progress`. Sub-managers + /// (`SpvRuntime`, `PlatformAddressSyncManager`, etc.) hold their + /// own clones already. + pub(super) event_manager: Arc, pub(super) persister: Arc

, /// Cancellation token + join handle for the wallet-event adapter /// task. Held so [`shutdown`] can stop it cleanly when the manager @@ -151,6 +158,7 @@ impl PlatformWalletManager

{ shielded_sync_manager: shielded_sync, #[cfg(feature = "shielded")] shielded_coordinator, + event_manager, persister, event_adapter_cancel, event_adapter_join: tokio::sync::Mutex::new(Some(event_adapter_join)), @@ -208,6 +216,33 @@ impl PlatformWalletManager

{ db_path, store, )); + // Bridge sync-internal chunk progress (~every 2048 notes) + // into the public `PlatformEventHandler::on_shielded_sync_progress` + // event so UI clients can render a live counter / progress + // bar during long cold syncs. Cheap closure — just forwards + // two u64s to the event manager. + let event_manager_for_progress = Arc::clone(&self.event_manager); + coordinator.install_progress_handler(Some(Arc::new( + move |cumulative_scanned: u64, block_height: u64| { + event_manager_for_progress + .on_shielded_sync_progress(cumulative_scanned, block_height); + }, + ))); + // Bridge sync-internal tree-commit progress (once per + // committed batch) into the public + // `PlatformEventHandler::on_shielded_tree_progress` event — the + // second "checked / committed-to-tree" signal, distinct from + // the "downloaded" counter above. `leaves_committed` is the + // cumulative tree leaf count; `total_target` is the on-chain + // MMR total (0 ⇒ indeterminate). Lets UI clients render a dual + // ProgressView ("downloaded" vs "checked") during cold syncs. + let event_manager_for_tree_progress = Arc::clone(&self.event_manager); + coordinator.install_tree_progress_handler(Some(Arc::new( + move |leaves_committed: u64, total_target: u64| { + event_manager_for_tree_progress + .on_shielded_tree_progress(leaves_committed, total_target); + }, + ))); *slot = Some(coordinator); Ok(()) } @@ -235,28 +270,45 @@ impl PlatformWalletManager

{ /// nothing can re-persist notes after this returns), then **clear** /// the network coordinator's per-subwallet registries. Idempotent — /// the coordinator step is a no-op when shielded support was never - /// configured. The per-network commitment-tree SQLite file is left - /// intact (chain-wide data; the next bind re-syncs against it). + /// configured. The per-network commitment-tree SQLite file stays on + /// disk but its contents are reset to empty so the next bind cold- + /// resyncs from index 0. + /// + /// Returns an error if the coordinator's store reset fails; the host + /// must not commit its own persistence wipe in that case. #[cfg(feature = "shielded")] - pub async fn clear_shielded(&self) { + pub async fn clear_shielded(&self) -> Result<(), crate::error::PlatformWalletError> { self.shielded_sync_manager.quiesce().await; if let Some(coord) = self.shielded_coordinator().await { - coord.clear().await; + coord.clear().await?; } + Ok(()) } /// Stop all background tasks and wait for them to exit. /// - /// Stops the periodic coordinators (`PlatformAddressSyncManager`, - /// `IdentitySyncManager`) and the wallet-event adapter task. + /// **Quiesces** the periodic coordinators + /// (`PlatformAddressSyncManager`, `IdentitySyncManager`, + /// `ShieldedSyncManager`) — cancelling each loop *and draining any + /// in-flight pass to completion*, including its persister / + /// host-callback fan-out — then drains the wallet-event adapter task. /// Idempotent. Call before dropping the manager when a clean /// shutdown is required (e.g. on app termination); a dirty drop /// simply leaks the tasks until the runtime exits. + /// + /// Ordering matters: cancel-only `stop()` would let a pass already + /// inside `sync_now` keep running and call `persister.store(...)` / + /// fire a host completion callback after the FFI's `destroy` + /// returned and the host freed the persister / event-handler + /// context — a use-after-free. So we `quiesce()` the sync managers + /// FIRST (so no further persister store or host callback can start), + /// and only THEN cancel + join the event adapter, which is the sink + /// those stores feed into. pub async fn shutdown(&self) { - self.platform_address_sync_manager.stop(); - self.identity_sync_manager.stop(); + self.platform_address_sync_manager.quiesce().await; + self.identity_sync_manager.quiesce().await; #[cfg(feature = "shielded")] - self.shielded_sync_manager.stop(); + self.shielded_sync_manager.quiesce().await; self.event_adapter_cancel.cancel(); if let Some(handle) = self.event_adapter_join.lock().await.take() { diff --git a/packages/rs-platform-wallet/src/manager/platform_address_sync.rs b/packages/rs-platform-wallet/src/manager/platform_address_sync.rs index d5e093d2ca4..e1a229806c2 100644 --- a/packages/rs-platform-wallet/src/manager/platform_address_sync.rs +++ b/packages/rs-platform-wallet/src/manager/platform_address_sync.rs @@ -99,6 +99,13 @@ pub struct PlatformAddressSyncManager { background_cancel: StdMutex>, interval_secs: AtomicU64, is_syncing: AtomicBool, + /// Set by [`quiesce`](Self::quiesce) to gate new passes while it + /// drains an in-flight one. `sync_now` bails (after taking the + /// `is_syncing` slot) when this is set, so once `quiesce` observes + /// `is_syncing == false` no further pass can start — giving shutdown + /// a real "no more host-visible sync-completed callbacks" barrier + /// that cancel-only [`stop`](Self::stop) does not provide. + quiescing: AtomicBool, /// Unix seconds of the last completed pass. `0` = never. last_sync_unix: AtomicU64, /// Shared config applied uniformly across wallets and accounts. @@ -120,6 +127,7 @@ impl PlatformAddressSyncManager { background_cancel: StdMutex::new(None), interval_secs: AtomicU64::new(DEFAULT_SYNC_INTERVAL_SECS), is_syncing: AtomicBool::new(false), + quiescing: AtomicBool::new(false), last_sync_unix: AtomicU64::new(0), config: ArcSwapOption::empty(), } @@ -224,6 +232,14 @@ impl PlatformAddressSyncManager { } /// Stop the background sync loop. No-op if not running. + /// + /// **Cancel-only**: requests cancellation and returns immediately. A + /// pass already inside `sync_now` keeps running to completion, + /// including its `on_platform_address_sync_completed` host-callback + /// dispatch. For a real "nothing is running and nothing more will + /// fire a host callback" barrier — required by manager shutdown so + /// the host can free the event-handler context — use + /// [`quiesce`](Self::quiesce). pub fn stop(&self) { if let Some(token) = self .background_cancel @@ -235,6 +251,34 @@ impl PlatformAddressSyncManager { } } + /// Cancel the background loop **and wait for any in-flight sync pass + /// to fully drain** before returning — a real quiescence barrier, + /// unlike cancel-only [`stop`](Self::stop). + /// + /// After this returns, no sync pass is running and none can start + /// until the next [`start`](Self::start) / `sync_now`, so a caller + /// that immediately tears the manager down (and frees the host-owned + /// event-handler context the FFI handed to us) cannot be raced by a + /// pass that fires `on_platform_address_sync_completed` through a + /// now-dangling pointer. + /// + /// Mechanism: set the `quiescing` gate so any pass that hasn't yet + /// taken the `is_syncing` slot bails, cancel the loop, then wait for + /// `is_syncing` to clear. `is_syncing` is held for the whole pass + /// including the completion-event dispatch (`sync_now` clears it only + /// after `on_platform_address_sync_completed` returns), so its + /// falling edge (with the gate up) is a sound "fully drained" signal. + /// The gate is reopened before returning so a later start/sync works + /// normally. + pub async fn quiesce(&self) { + self.quiescing.store(true, Ordering::Release); + self.stop(); + while self.is_syncing.load(Ordering::Acquire) { + tokio::time::sleep(Duration::from_millis(20)).await; + } + self.quiescing.store(false, Ordering::Release); + } + /// Run one sync pass across every registered wallet. /// /// If a pass is already in flight, returns an empty summary and @@ -248,6 +292,16 @@ impl PlatformAddressSyncManager { return PlatformAddressSyncSummary::default(); } + // A `quiesce()` may have raised the gate between our CAS and + // here; if so, release the slot and bail without running a pass + // so the drain can complete and shutdown gets a true barrier + // (no further `on_platform_address_sync_completed` host callback + // after quiesce returns). + if self.quiescing.load(Ordering::Acquire) { + self.is_syncing.store(false, Ordering::Release); + return PlatformAddressSyncSummary::default(); + } + let snapshot: Vec<(WalletId, Arc)> = { let wallets = self.wallets.read().await; wallets.iter().map(|(id, w)| (*id, Arc::clone(w))).collect() @@ -277,11 +331,20 @@ impl PlatformAddressSyncManager { .unwrap_or(0); summary.sync_unix_seconds = now; self.last_sync_unix.store(now, Ordering::Release); - self.is_syncing.store(false, Ordering::Release); + // Dispatch the completion event BEFORE clearing `is_syncing`. + // `quiesce()` drains on the falling edge of `is_syncing`, so if + // we cleared the flag first a shutdown caller could unblock and + // free the host event-handler context while this completion + // event (FFI callback → host handler) is still pending — a + // use-after-free. Holding the flag across the dispatch makes + // quiesce's barrier cover the host callback too. Mirrors the + // ordering in `ShieldedSyncManager::sync_now`. self.event_manager .on_platform_address_sync_completed(&summary); + self.is_syncing.store(false, Ordering::Release); + summary } @@ -315,3 +378,131 @@ impl std::fmt::Debug for PlatformAddressSyncManager { .finish() } } + +#[cfg(test)] +mod tests { + use super::*; + + use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; + + use dash_spv::EventHandler; + + use crate::events::PlatformEventHandler; + + /// Event handler that just counts `on_platform_address_sync_completed` + /// dispatches. Stands in for the host's FFI handler so we can assert + /// the quiescing gate suppresses the completion callback. + struct CompletionCounter { + completions: AtomicUsize, + } + + impl CompletionCounter { + fn new() -> Self { + Self { + completions: AtomicUsize::new(0), + } + } + } + + impl EventHandler for CompletionCounter {} + + impl PlatformEventHandler for CompletionCounter { + fn on_platform_address_sync_completed(&self, _summary: &PlatformAddressSyncSummary) { + self.completions.fetch_add(1, AtomicOrdering::SeqCst); + } + } + + /// Build a manager over an empty wallet map wired to a completion + /// counter. No wallets means `sync_now` runs zero per-wallet syncs + /// but still drives the full flag → gate → completion-event protocol + /// we're testing here. + fn make_manager() -> (Arc, Arc) { + let wallets = Arc::new(RwLock::new(BTreeMap::new())); + let counter = Arc::new(CompletionCounter::new()); + let event_manager = Arc::new(PlatformEventManager::new(vec![ + Arc::clone(&counter) as Arc + ])); + ( + Arc::new(PlatformAddressSyncManager::new(wallets, event_manager)), + counter, + ) + } + + /// A normal pass (no gate) fires the completion event and leaves the + /// flags clean. Baseline for the gated case below. + #[tokio::test] + async fn sync_now_fires_completion_when_not_quiescing() { + let (mgr, counter) = make_manager(); + mgr.sync_now().await; + assert_eq!(counter.completions.load(AtomicOrdering::SeqCst), 1); + assert!(!mgr.is_syncing()); + } + + /// `quiesce()` must not return while a pass is in flight, and must + /// return promptly once the pass drains. + /// + /// Drives the real `is_syncing` lifecycle: a background task takes + /// the slot via the same `compare_exchange` the real `sync_now` + /// uses, holds it across a sleep (standing in for the pass body + + /// completion-event dispatch, which `sync_now` keeps the flag set + /// across), then clears it. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn quiesce_blocks_until_in_flight_pass_drains() { + let (mgr, _counter) = make_manager(); + + let holder = Arc::clone(&mgr); + let pass = tokio::spawn(async move { + assert!( + holder + .is_syncing + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok(), + "test should own the is_syncing slot" + ); + tokio::time::sleep(Duration::from_millis(200)).await; + holder.is_syncing.store(false, Ordering::Release); + }); + + while !mgr.is_syncing() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + + let quiesce_fut = mgr.quiesce(); + tokio::pin!(quiesce_fut); + + tokio::select! { + _ = &mut quiesce_fut => panic!("quiesce returned while a pass was in flight"), + _ = tokio::time::sleep(Duration::from_millis(50)) => {} + } + assert!(mgr.is_syncing(), "pass should still be in flight"); + + tokio::time::timeout(Duration::from_secs(2), &mut quiesce_fut) + .await + .expect("quiesce did not return after the pass drained"); + + assert!(!mgr.quiescing.load(Ordering::Acquire)); + assert!(!mgr.is_syncing()); + pass.await.unwrap(); + } + + /// A `sync_now()` invoked while `quiescing` is set must bail without + /// running the pass — in particular, without firing the + /// `on_platform_address_sync_completed` host callback. This is the + /// gate that prevents a pass from slipping in between `quiesce`'s + /// `stop()` and its drain. + #[tokio::test] + async fn sync_now_bails_when_quiescing() { + let (mgr, counter) = make_manager(); + + // Raise the gate as `quiesce()` would. + mgr.quiescing.store(true, Ordering::Release); + + let summary = mgr.sync_now().await; + + // Empty summary, no host completion callback, slot released so a + // later (post-quiesce) pass can still run. + assert!(summary.is_empty()); + assert_eq!(counter.completions.load(AtomicOrdering::SeqCst), 0); + assert!(!mgr.is_syncing()); + } +} diff --git a/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs b/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs index 98aec94dc71..b78faa60a9f 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs @@ -54,6 +54,26 @@ use std::collections::BTreeMap; use std::path::PathBuf; use std::sync::Arc; + +/// Callback fired once per chunk during a coordinator sync pass — +/// the **"downloaded"** progress signal. +/// Arguments: `(cumulative_scanned, latest_block_height)`. Forwarded +/// straight from the SDK stream's per-chunk download completion. +pub type ShieldedProgressCallback = Arc; + +/// Callback fired as commitments are committed to the coordinator's +/// local Merkle tree — the **"checked / committed-to-tree"** progress +/// signal, distinct from [`ShieldedProgressCallback`] (network +/// download). Fired once per appended batch during the interleaved +/// stream consume in [`sync_notes_across`]. +/// +/// Arguments: `(cumulative_leaves_committed, total_leaves_target)`. +/// `total_leaves_target` is the on-chain MMR leaf count fetched once at +/// the start of the pass; it is `0` when that progress-only RPC failed, +/// which the UI should treat as an indeterminate total. +/// +/// [`sync_notes_across`]: super::sync::sync_notes_across +pub type ShieldedTreeProgressCallback = Arc; use std::time::{Duration, Instant}; use tokio::sync::RwLock; @@ -137,6 +157,30 @@ pub struct NetworkShieldedCoordinator { /// network instead of once per wallet. Cleared on any /// activity; bypassed by `force` syncs. last_caught_up_at: std::sync::Mutex>, + + /// Optional progress callback fired once per chunk inside + /// `sync_shielded_notes`. Lets the manager translate chunk-level + /// progress into `PlatformEventHandler::on_shielded_sync_progress` + /// events without sync_notes_across knowing about the event + /// manager. Installed by the manager via + /// [`install_progress_handler`](Self::install_progress_handler); + /// `None` (default) disables progress reporting (the test path). + /// + /// `std::sync::Mutex` rather than `ArcSwap` because `arc_swap` + /// requires `T: Sized` and we need to hold a `dyn Fn`. The lock + /// is taken once per sync pass to read the snapshot — no hot-path + /// contention. + progress_handler: std::sync::Mutex>, + + /// Optional tree-progress callback fired as commitments are + /// committed to the local Merkle tree during the interleaved sync + /// (the "checked" signal, distinct from `progress_handler`'s + /// "downloaded" signal). Installed by the manager via + /// [`install_tree_progress_handler`](Self::install_tree_progress_handler); + /// `None` (default) disables tree-progress reporting. + /// + /// Same `std::sync::Mutex` rationale as `progress_handler`. + tree_progress_handler: std::sync::Mutex>, } impl NetworkShieldedCoordinator { @@ -161,6 +205,8 @@ impl NetworkShieldedCoordinator { accounts: Arc::new(RwLock::new(BTreeMap::new())), persisters: Arc::new(RwLock::new(BTreeMap::new())), last_caught_up_at: std::sync::Mutex::new(None), + progress_handler: std::sync::Mutex::new(None), + tree_progress_handler: std::sync::Mutex::new(None), } } @@ -171,6 +217,45 @@ impl NetworkShieldedCoordinator { self.network } + /// Install (or replace) the per-chunk progress handler. The + /// callback runs from inside `sync_shielded_notes`'s chunk loop + /// — once per ~2048 notes processed — so keep it cheap. Used by + /// `PlatformWalletManager` to bridge sync-internal progress into + /// `PlatformEventHandler::on_shielded_sync_progress` events. + /// Passing `None` removes any installed handler. + pub fn install_progress_handler(&self, handler: Option) { + if let Ok(mut slot) = self.progress_handler.lock() { + *slot = handler; + } + } + + /// Snapshot of the currently installed progress handler. + pub(super) fn progress_handler(&self) -> Option { + self.progress_handler.lock().ok().and_then(|g| g.clone()) + } + + /// Install (or replace) the tree-progress handler — the "checked / + /// committed-to-tree" signal fired as commitments are appended to + /// the local Merkle tree during the interleaved sync. Fired once per + /// appended batch (~8192-note batches), so it's already coarse; + /// still keep the callback cheap. Used by `PlatformWalletManager` + /// to bridge tree progress into a second progress bar, distinct + /// from the download progress handler. Passing `None` removes any + /// installed handler. + pub fn install_tree_progress_handler(&self, handler: Option) { + if let Ok(mut slot) = self.tree_progress_handler.lock() { + *slot = handler; + } + } + + /// Snapshot of the currently installed tree-progress handler. + pub(super) fn tree_progress_handler(&self) -> Option { + self.tree_progress_handler + .lock() + .ok() + .and_then(|g| g.clone()) + } + /// The on-disk SQLite path the coordinator opened. Used by /// `PlatformWalletManager::configure_shielded` to verify /// subsequent calls pass the same path. @@ -336,11 +421,11 @@ impl NetworkShieldedCoordinator { /// Drop every wallet registration, purge all per-subwallet /// store state (notes, spent marks, sync watermarks, - /// nullifier checkpoints), and reset the cooldown stamp. The - /// single SQLite handle (commitment tree) stays open — Clear - /// semantics on the host side are "wipe my persistence and - /// start re-syncing from index 0 on the shared tree", not - /// "blow away the chain-wide cache". + /// nullifier checkpoints), empty the shared commitment tree, + /// and reset the cooldown stamp. The single SQLite handle stays + /// open — Clear semantics on the host side are "wipe my + /// persistence and cold-rebuild from index 0", not "blow away + /// the SQLite file". /// /// Purging the in-memory `subwallets` store is what actually /// delivers the "re-sync from index 0" contract: the sync @@ -351,6 +436,18 @@ impl NetworkShieldedCoordinator { /// only work after a process restart that drops the /// in-memory state). Clearing it here closes that gap. /// + /// Emptying the commitment tree is what keeps the two reset + /// halves coherent. The watermark rewinds to 0 but the tree's + /// append gate is `tree_size`, so a tree left at its full + /// (~1M-leaf) size would gate-skip every re-downloaded position + /// (`global_pos < tree_size`) — nothing new appends, the + /// "Checked" progress bar stays pinned at the stale leaf count + /// while "Downloaded" climbs from 0, and the host pointlessly + /// re-downloads into an already-complete tree. Resetting the + /// tree alongside the watermarks makes Clear+resync a true cold + /// rebuild: `tree_size` returns to 0 and "Checked" climbs 0→N + /// trailing "Downloaded". + /// /// Used by [`platform_wallet_manager_shielded_clear`] (the /// host's Clear button). The host then wipes its own /// per-wallet persistence (e.g. SwiftData rows) — Rust can't @@ -363,15 +460,62 @@ impl NetworkShieldedCoordinator { /// /// [`platform_wallet_manager_shielded_clear`]: /// rs-platform-wallet-ffi's FFI entry point - pub async fn clear(&self) { + /// + /// Returns an error if either store reset (subwallet purge or + /// commitment-tree reset) fails. The caller **must** surface this: + /// the host only wipes its own per-wallet persistence (e.g. + /// SwiftData rows) after `clear()` succeeds. If a reset fails + /// silently the host could drop its rows while the shared tree + /// stays populated, and the next cold resync would gate-skip every + /// re-downloaded position against the stale `tree_size`. + pub async fn clear(&self) -> Result<(), crate::error::PlatformWalletError> { + // Reset the persistent store FIRST and bail before mutating any + // in-memory state if it fails. Clearing `accounts` / `persisters` + // makes the coordinator forget every bound wallet (no syncs until + // the host rebinds), so doing that while the store reset failed — + // and the host therefore keeps its own local state — would leave + // the two halves inconsistent. Both resets are still attempted + // even if the first fails, so the store is left as clean as + // possible, but the first error is captured and propagated. + let mut first_err: Option = None; + { + let mut store = self.store.write().await; + if let Err(e) = store.purge_all_subwallets() { + tracing::warn!(error = %e, "Failed to purge subwallet store state on clear"); + first_err.get_or_insert_with(|| { + crate::error::PlatformWalletError::ShieldedStoreError(format!( + "purge_all_subwallets failed: {e}" + )) + }); + } + // Reset the shared commitment tree under the same write + // guard so the watermark (now 0) and the tree size reset + // together — otherwise the post-clear resync gate-skips + // every re-downloaded position into the still-full tree. + if let Err(e) = store.reset_commitment_tree() { + tracing::warn!(error = %e, "Failed to reset commitment tree on clear"); + first_err.get_or_insert_with(|| { + crate::error::PlatformWalletError::ShieldedStoreError(format!( + "reset_commitment_tree failed: {e}" + )) + }); + } + } + if let Some(e) = first_err { + return Err(e); + } + + // Store reset succeeded — now it is safe to drop the in-memory + // registries and reset the cooldown so the first post-clear + // background pass runs immediately rather than honoring a stale + // "caught up" stamp. On the failure path above none of this runs, + // so a failed clear leaves coordinator state untouched. self.accounts.write().await.clear(); self.persisters.write().await.clear(); - if let Err(e) = self.store.write().await.purge_all_subwallets() { - tracing::warn!(error = %e, "Failed to purge subwallet store state on clear"); - } if let Ok(mut g) = self.last_caught_up_at.lock() { *g = None; } + Ok(()) } /// Run one shielded sync pass for every registered wallet on @@ -437,7 +581,23 @@ impl NetworkShieldedCoordinator { } // ONE SDK call covers every registered IVK on the network. - let notes = match super::sync::sync_notes_across(&self.sdk, &self.store, &subwallets).await + // Snapshot the optional progress handler installed by the + // manager; sync_notes_across feeds it into the SDK's chunk + // loop so callers see live (cumulative_scanned, block_height) + // updates during long cold syncs instead of one delayed + // burst at the end. + let on_progress = self.progress_handler(); + // Second, distinct signal: commitments committed to the local + // tree as the interleaved consumer drains the SDK stream. + let on_tree_progress = self.tree_progress_handler(); + let notes = match super::sync::sync_notes_across( + &self.sdk, + &self.store, + &subwallets, + on_progress.as_ref(), + on_tree_progress.as_ref(), + ) + .await { Ok(r) => r, Err(e) => return self.fail_all_wallets(&subwallets, &e), @@ -640,3 +800,115 @@ fn build_per_wallet_summary( } summary } + +#[cfg(test)] +mod tests { + use super::*; + use crate::wallet::persister::NoPlatformPersistence; + use crate::wallet::shielded::keys::OrchardKeySet; + + /// Unique temp directory for a test's SQLite tree (no `tempfile` dev-dep). + fn temp_dir(tag: &str) -> PathBuf { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + std::env::temp_dir().join(format!("shielded_coordinator_test_{tag}_{nanos}")) + } + + /// Build a coordinator backed by a fresh file store under `dir`, with one + /// wallet registered so `accounts` / `persisters` are both non-empty. + async fn coordinator_with_one_wallet(dir: &std::path::Path) -> NetworkShieldedCoordinator { + std::fs::create_dir_all(dir).expect("create temp dir"); + let db_path = dir.join("tree.sqlite"); + let store = FileBackedShieldedStore::open_path(&db_path, 100).expect("open file store"); + let coordinator = NetworkShieldedCoordinator::new( + Arc::new(dash_sdk::Sdk::new_mock()), + dashcore::Network::Testnet, + db_path, + store, + ); + + let wallet_id: WalletId = [0x11; 32]; + let views = OrchardKeySet::from_seed(&[0x42u8; 64], dashcore::Network::Testnet, 0) + .expect("derive viewing keys") + .viewing_keys(); + let mut account_views = BTreeMap::new(); + account_views.insert(0u32, views); + let persister = WalletPersister::new(wallet_id, Arc::new(NoPlatformPersistence)); + coordinator + .register_wallet(wallet_id, account_views, persister) + .await; + coordinator + } + + /// Success path: a healthy `clear()` empties the shared commitment tree + /// AND drops the in-memory account / persister registries, returning `Ok`. + #[tokio::test] + async fn clear_success_empties_tree_and_registries() { + let dir = temp_dir("clear_ok"); + let coordinator = coordinator_with_one_wallet(&dir).await; + + // Put a leaf in the tree so the reset is observable. + { + let mut store = coordinator.store().write().await; + store.append_commitment(&[7u8; 32], true).unwrap(); + assert_eq!(store.tree_size().unwrap(), 1); + } + assert!(!coordinator.accounts.read().await.is_empty()); + assert!(!coordinator.persisters.read().await.is_empty()); + + coordinator.clear().await.expect("clear should succeed"); + + assert_eq!( + coordinator.store().read().await.tree_size().unwrap(), + 0, + "tree must be empty after a successful clear" + ); + assert!(coordinator.accounts.read().await.is_empty()); + assert!(coordinator.persisters.read().await.is_empty()); + + let _ = std::fs::remove_dir_all(&dir); + } + + /// Failure path — regression guard for the partial-clear bug: when the + /// store reset fails, `clear()` must return `Err` and leave the account / + /// persister registries populated, so the coordinator does not silently + /// forget every bound wallet (turning future syncs into no-ops) while the + /// host is told to keep its own persisted state. + /// + /// Unix-only: the failure injection relies on POSIX unlink-while-open + /// semantics — removing the directory orphans the inode the store's open + /// SQLite handle keeps using, but a fresh `Connection::open` at the now + /// missing path fails, which is what drives `reset_commitment_tree` to + /// error. Windows refuses to remove a directory with open files, so the + /// injection wouldn't model a reset failure there. + #[cfg(unix)] + #[tokio::test] + async fn clear_failure_preserves_registries() { + let dir = temp_dir("clear_err"); + let coordinator = coordinator_with_one_wallet(&dir).await; + assert!(!coordinator.accounts.read().await.is_empty()); + assert!(!coordinator.persisters.read().await.is_empty()); + + // Force `reset_commitment_tree` to fail: remove the directory holding + // the SQLite file so the reset's `Connection::open(path)` can't reopen + // it. The store's already-open handle keeps working on the unlinked + // inode, but a fresh open at the now-missing path errors. + std::fs::remove_dir_all(&dir).expect("remove temp dir to break reopen"); + + let result = coordinator.clear().await; + assert!( + result.is_err(), + "clear must surface the store-reset failure rather than swallow it" + ); + assert!( + !coordinator.accounts.read().await.is_empty(), + "accounts must survive a failed clear so sync does not become a no-op" + ); + assert!( + !coordinator.persisters.read().await.is_empty(), + "persisters must survive a failed clear" + ); + } +} diff --git a/packages/rs-platform-wallet/src/wallet/shielded/file_store.rs b/packages/rs-platform-wallet/src/wallet/shielded/file_store.rs index 6eaf1803675..bd325ee1d42 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/file_store.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/file_store.rs @@ -14,7 +14,7 @@ use std::collections::BTreeMap; use std::error::Error as StdError; use std::fmt; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Mutex; use grovedb_commitment_tree::{ClientPersistentCommitmentTree, Position, Retention}; @@ -45,6 +45,21 @@ pub struct FileBackedShieldedStore { /// `RwLock`; this inner mutex is just a `Sync`-restoring /// shim and is uncontended in practice. tree: Mutex, + /// Backing SQLite path, retained so [`reset_commitment_tree`] + /// can wipe the on-disk tree tables and rebuild a fresh + /// `ClientPersistentCommitmentTree` over the same file. The + /// wrapper takes its `Connection` by value and exposes no + /// public truncate, so a full reset reopens the tree rather + /// than mutating the live handle in place. + /// + /// [`reset_commitment_tree`]: ShieldedStore::reset_commitment_tree + path: PathBuf, + /// `max_checkpoints` passed at open time, retained so the + /// rebuilt tree in [`reset_commitment_tree`] matches the + /// original retention policy. + /// + /// [`reset_commitment_tree`]: ShieldedStore::reset_commitment_tree + max_checkpoints: usize, /// Per-subwallet notes + sync state, keyed by `(wallet_id, /// account_index)`. Lazily populated on first use of an id. subwallets: BTreeMap, @@ -52,17 +67,65 @@ pub struct FileBackedShieldedStore { impl FileBackedShieldedStore { /// Open or create a shielded store at `path`. + /// + /// SQLite is opened with **WAL journal + synchronous=NORMAL + temp_store=MEMORY** + /// rather than the rusqlite defaults (DELETE + sync=FULL). Rationale: every + /// `append_commitment` invocation runs an implicit one-statement transaction + /// that, under DELETE+FULL, forces a fsync per cmx. On hosts where fsync is + /// strictly honored (macOS Mac/simulator filesystems), that turns into the + /// dominant cost of cold sync — a 1M-leaf tree build was ~6 min, vs ~17 s + /// with the PRAGMAs below, per + /// `packages/rs-platform-wallet/tests/shielded_tree_append_bench.rs`. + /// + /// `synchronous=NORMAL` retains crash-safety for the WAL (the WAL itself is + /// fsync'd at checkpoint); we don't need `FULL` because no row in the + /// commitment-tree SQLite is "user money" — every commitment is chain-side + /// authenticated and can be rebuilt by re-running sync from a recorded + /// `last_synced_note_index`. A torn WAL on power loss would at worst + /// require resync from the last checkpoint, which is the same cost the + /// host already accepts on a fresh install. pub fn open_path( path: impl AsRef, max_checkpoints: usize, ) -> Result { - let tree = ClientPersistentCommitmentTree::open_path(path, max_checkpoints) + let path = path.as_ref().to_path_buf(); + let conn = Self::open_tuned_connection(&path)?; + let tree = ClientPersistentCommitmentTree::open(conn, max_checkpoints) .map_err(|e| FileShieldedStoreError(format!("open commitment tree: {e}")))?; Ok(Self { tree: Mutex::new(tree), + path, + max_checkpoints, subwallets: BTreeMap::new(), }) } + + /// Open a `rusqlite::Connection` on `path` with the same WAL / + /// `synchronous=NORMAL` / `temp_store=MEMORY` PRAGMAs the cold-sync + /// append path depends on (see [`open_path`] for the rationale). + /// + /// Shared by [`open_path`] and [`reset_commitment_tree`] so any + /// connection the store hands to `ClientPersistentCommitmentTree` + /// — original or post-reset — is configured identically. + /// + /// [`open_path`]: Self::open_path + /// [`reset_commitment_tree`]: ShieldedStore::reset_commitment_tree + fn open_tuned_connection(path: &Path) -> Result { + let conn = rusqlite::Connection::open(path) + .map_err(|e| FileShieldedStoreError(format!("open sqlite: {e}")))?; + // Pragmas must be applied before the schema is touched. They survive + // for the lifetime of the connection; WAL also persists for any + // subsequent reopen on the same file until explicitly changed. + for (k, v) in [ + ("journal_mode", "WAL"), + ("synchronous", "NORMAL"), + ("temp_store", "MEMORY"), + ] { + conn.pragma_update(None, k, v) + .map_err(|e| FileShieldedStoreError(format!("PRAGMA {k}={v}: {e}")))?; + } + Ok(conn) + } } impl ShieldedStore for FileBackedShieldedStore { @@ -225,6 +288,42 @@ impl ShieldedStore for FileBackedShieldedStore { self.subwallets.clear(); Ok(()) } + + fn reset_commitment_tree(&mut self) -> Result<(), Self::Error> { + // The `ClientPersistentCommitmentTree` wrapper owns its + // `Connection` and exposes no public truncate (only the inner + // `SqliteShardStore` has `truncate_shards`). A full reset + // therefore (1) wipes the four `commitment_tree_*` tables on a + // fresh connection, then (2) rebuilds the wrapper over the now + // empty DB so the in-memory shardtree frontier/cap reflect the + // empty state. Reopening — rather than mutating the live tree — + // is what guarantees `tree_size()` reads back 0: the wrapper + // caches frontier nodes that a bare `DELETE` wouldn't clear. + let mut tree = self + .tree + .lock() + .map_err(|e| FileShieldedStoreError(format!("tree mutex poisoned: {e}")))?; + + { + let conn = Self::open_tuned_connection(&self.path)?; + // `commitment_tree_cap` is included alongside the three + // shard/checkpoint tables: it caches upper-level tree nodes, + // so leaving it populated while the shards are empty would + // reopen into an inconsistent (non-empty) tree state. + conn.execute_batch( + "DELETE FROM commitment_tree_checkpoint_marks_removed; + DELETE FROM commitment_tree_checkpoints; + DELETE FROM commitment_tree_shards; + DELETE FROM commitment_tree_cap;", + ) + .map_err(|e| FileShieldedStoreError(format!("reset commitment tree tables: {e}")))?; + } + + let conn = Self::open_tuned_connection(&self.path)?; + *tree = ClientPersistentCommitmentTree::open(conn, self.max_checkpoints) + .map_err(|e| FileShieldedStoreError(format!("reopen commitment tree: {e}")))?; + Ok(()) + } } #[cfg(test)] @@ -341,4 +440,67 @@ mod tests { reads it on cold start to avoid re-appending existing leaves" ); } + + /// `reset_commitment_tree()` must empty the shared tree back to + /// zero leaves so the host's "Clear" action becomes a true cold + /// rebuild: after a reset, `tree_size()` is 0, a fresh append + /// starts at position 0, and the emptied state survives a + /// persist + reload (the on-disk tables are genuinely wiped, not + /// just the in-memory frontier). Without this, Clear rewinds the + /// per-subwallet watermark to 0 but leaves the tree at its full + /// size, so every re-downloaded position is gate-skipped and the + /// "Checked" progress bar stalls. + #[test] + fn reset_commitment_tree_empties_and_allows_reappend_from_zero() { + let path = temp_tree_path("reset"); + let mut store = FileBackedShieldedStore::open_path(&path, 100).unwrap(); + + // Build a non-trivial tree. + const N: u64 = 6; + for i in 0..N { + let mut cmx = [0u8; 32]; + cmx[0] = (i as u8) + 1; + store.append_commitment(&cmx, true).unwrap(); + } + store.checkpoint_tree(N as u32).unwrap(); + assert_eq!( + store.tree_size().unwrap(), + N, + "precondition: tree holds N leaves before reset" + ); + + // Reset wipes it back to empty. + store.reset_commitment_tree().unwrap(); + assert_eq!( + store.tree_size().unwrap(), + 0, + "tree_size must be 0 immediately after reset" + ); + + // A fresh append starts at position 0 again and the count + // climbs from there — the cold-rebuild contract Clear relies on. + let mut cmx = [0u8; 32]; + cmx[0] = 42; + store.append_commitment(&cmx, true).unwrap(); + assert_eq!( + store.tree_size().unwrap(), + 1, + "first post-reset append must land at position 0 (size 1)" + ); + store.checkpoint_tree(1).unwrap(); + + // The emptied + re-appended state must survive persist + + // reload, proving the reset wiped the on-disk tables rather + // than only the in-memory frontier. + drop(store); + let store = FileBackedShieldedStore::open_path(&path, 100).unwrap(); + let size = store.tree_size().unwrap(); + let _ = std::fs::remove_file(&path); + + assert_eq!( + size, 1, + "post-reset tree state (1 leaf) must survive persist + reload, \ + confirming reset cleared the SQLite tree tables" + ); + } } diff --git a/packages/rs-platform-wallet/src/wallet/shielded/store.rs b/packages/rs-platform-wallet/src/wallet/shielded/store.rs index 038f691cd8b..0950fcd7c77 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/store.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/store.rs @@ -217,6 +217,24 @@ pub trait ShieldedStore: Send + Sync { /// of every wallet. The shared commitment tree is left /// untouched. Used by `NetworkShieldedCoordinator::clear()`. fn purge_all_subwallets(&mut self) -> Result<(), Self::Error>; + + /// Empty the shared commitment tree back to zero leaves. + /// + /// After this returns, [`Self::tree_size`] reports `0` and the + /// next [`Self::append_commitment`] starts at position `0`. The + /// per-subwallet watermarks ([`Self::last_synced_note_index`]) + /// are *not* touched here — callers that want a cold rebuild + /// pair this with [`Self::purge_all_subwallets`] so the + /// re-download watermark and the tree reset together. + /// + /// Used by `NetworkShieldedCoordinator::clear()` so the host's + /// "Clear" action is a true cold reset rather than a watermark + /// rewind into an already-full tree. Without it, Clear leaves + /// the tree at its full size while the watermark drops to 0, so + /// every re-fetched position is gate-skipped (`global_pos < + /// tree_size`) and the "Checked" progress bar stays pinned at + /// the stale leaf count while "Downloaded" climbs from 0. + fn reset_commitment_tree(&mut self) -> Result<(), Self::Error>; } // ── Per-subwallet bookkeeping ────────────────────────────────────────── @@ -464,6 +482,18 @@ impl ShieldedStore for InMemoryShieldedStore { self.subwallets.clear(); Ok(()) } + + fn reset_commitment_tree(&mut self) -> Result<(), Self::Error> { + // Flat-list backing: clearing the commitment / mark / + // checkpoint vectors drops `tree_size()` to 0 and makes the + // next append start at position 0, matching the file-backed + // store's reset contract. + self.commitments.clear(); + self.marked_positions.clear(); + self.checkpoints.clear(); + self.anchor = [0u8; 32]; + Ok(()) + } } #[cfg(test)] @@ -552,4 +582,24 @@ mod tests { store.checkpoint_tree(1).unwrap(); assert_eq!(store.tree_anchor().unwrap(), [0u8; 32]); } + + #[test] + fn test_reset_commitment_tree_empties_and_reappends_from_zero() { + let mut store = InMemoryShieldedStore::new(); + store.append_commitment(&[1u8; 32], true).unwrap(); + store.append_commitment(&[2u8; 32], true).unwrap(); + store.checkpoint_tree(2).unwrap(); + assert_eq!(store.tree_size().unwrap(), 2); + + store.reset_commitment_tree().unwrap(); + assert_eq!( + store.tree_size().unwrap(), + 0, + "tree_size must be 0 after reset" + ); + + // Re-append starts from position 0 again. + store.append_commitment(&[3u8; 32], true).unwrap(); + assert_eq!(store.tree_size().unwrap(), 1); + } } diff --git a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs index 146a2f4466a..451a3ca71b7 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs @@ -28,7 +28,8 @@ use std::collections::BTreeMap; use std::sync::Arc; use dash_sdk::platform::shielded::nullifier_sync::{NullifierSyncCheckpoint, NullifierSyncConfig}; -use dash_sdk::platform::shielded::{sync_shielded_notes, try_decrypt_note}; +use dash_sdk::platform::shielded::{sync_shielded_notes_stream, try_decrypt_note}; +use futures::StreamExt; use grovedb_commitment_tree::{Note as OrchardNote, PaymentAddress}; use tokio::sync::RwLock; use tracing::{debug, info, warn}; @@ -178,6 +179,8 @@ pub(super) async fn sync_notes_across( sdk: &Arc, store: &Arc>, subwallets: &[(SubwalletId, AccountViewingKeys)], + on_progress: Option<&super::coordinator::ShieldedProgressCallback>, + on_tree_progress: Option<&super::coordinator::ShieldedTreeProgressCallback>, ) -> Result { if subwallets.is_empty() { return Ok(MultiSyncNotesResult::default()); @@ -222,77 +225,97 @@ pub(super) async fn sync_notes_across( already_have, aligned_start, tree_size, "Starting multi-subwallet shielded note sync" ); - // Fetch + trial-decrypt with the FIRST subwallet's IVK in - // one SDK call. The driver's hits come back as - // `result.decrypted_notes`; every other subwallet's are - // produced by local trial-decryption against - // `result.all_notes` below. + // Denominator for the tree-progress ("checked") bar: the on-chain + // total leaf count of the shielded `CommitmentTree`. This now comes + // from the note-fetch proofs themselves — every streamed batch carries + // `total_count`, extracted from the SAME proof that delivers the notes + // (the parent CommitmentTree element is always present in it). That + // removes the separate up-front `GetShieldedNotesCount` RPC: the + // denominator works against currently-deployed nodes with no dependency + // on that RPC being deployed, and costs no extra round-trip. + // + // Seeded from the local `tree_size`, not 0: the on-chain count is + // append-only, so the real total can never legitimately fall below what + // we have already committed locally, and both numerators are floored at + // `tree_size` (the download clamp below and `leaves_committed`'s seed). + // Starting at 0 would let a first batch proven by a lagging node report + // `total_count < tree_size`, making the numerators briefly read above the + // denominator (progress > 100%) until a fresher chunk arrives. It is then + // raised to `batch.total_count` as batches land. On a cold sync + // `tree_size == 0`, so this preserves the host's indeterminate handling + // (total 0) until the first batch lands. + let mut total_target: u64 = tree_size; + + // Drive the FIRST subwallet's IVK as the streaming driver. Its hits + // come back per batch as `batch.decrypted`; every other subwallet's + // are produced by local trial-decryption against `batch.notes`. let (driver_id, driver_views) = &subwallets[0]; let driver_ivk = driver_views.prepared_ivk.clone(); - let result = sync_shielded_notes(sdk, &driver_ivk, aligned_start, None) - .await - .map_err(|e| PlatformWalletError::ShieldedSyncFailed(e.to_string()))?; - - info!( - total_scanned = result.total_notes_scanned, - decrypted_for_driver = result.decrypted_notes.len(), - next_start_index = result.next_start_index, - "SDK sync returned" - ); - - if result.next_start_index == 0 && result.total_notes_scanned > 0 { - warn!( - "Shielded sync: next_start_index is 0 after scanning {} notes — \ - next sync will rescan from the beginning", - result.total_notes_scanned, - ); + // Network-only config carrying the caller's "downloaded" progress + // callback; the SDK fires it once per completed network chunk inside + // the stream. The tree-progress ("checked") callback is owned by + // this consumer and fired below — it never travels through the SDK + // config (the SDK doesn't append to a tree). + // Fetch up to 16 chunks concurrently (default is 4) to parallelize + // across the network's ~13 nodes. The per-chunk cost is dominated by + // server-side proof generation, so more in-flight requests is the main + // client-side lever; the pull-based stream still caps in-flight fetches + // at this bound and keeps memory bounded. + let mut sync_config = dash_sdk::platform::shielded::notes_sync::types::ShieldedSyncConfig { + max_concurrent: 16, + ..Default::default() + }; + if let Some(cb) = on_progress { + // Clamp the SDK's "downloaded" value up to the pre-stream tree leaf + // count before forwarding it to the host. The SDK reports downloaded + // as `aligned_start + scanned`, where `aligned_start` is the rewound + // MIN watermark across subwallets and can sit far below `tree_size` + // (a second subwallet binding at watermark 0 collapses it to 0; a + // partial-chunk-tail resume rewinds it below the tail). The "checked" + // signal is the absolute tree size, so over the gate-skipped re-scan + // region (`global_pos < tree_size`, no new appends) the unclamped + // download value reads *below* checked and breaks the advertised + // `Checked ≤ Downloaded ≤ total` invariant. Everything already in the + // tree was necessarily downloaded on a prior pass, so counting it as + // downloaded is accurate; the clamp is a no-op once the stream + // advances past `tree_size`, and the cold-sync case (tree_size == 0) + // is entirely unaffected. + let cb = cb.clone(); + let tree_baseline = tree_size; + sync_config.on_chunk_completed = Some(Arc::new(move |downloaded: u64, height: u64| { + cb(downloaded.max(tree_baseline), height); + })); } - // Route decryptions to the subwallet that owns the IVK. - let mut decrypted_by_subwallet: BTreeMap> = BTreeMap::new(); - for dn in &result.decrypted_notes { - decrypted_by_subwallet - .entry(*driver_id) - .or_default() - .push(DiscoveredNote { - position: dn.position, - cmx: dn.cmx, - note: dn.note, - }); - } + // Acquire the store write lock for the whole interleaved consume. + // This is the only writer during a pass, so holding it across + // `stream.next().await` is safe: the stream's producer side is the + // SDK's network fetch loop, which never touches the store, so there + // is no lock-ordering deadlock. Backpressure is pull-based — a slow + // append simply polls the stream less often, capping in-flight + // network fetches at `max_concurrent`. + let mut store = store.write().await; - for (id, views) in subwallets.iter().skip(1) { - for (i, raw_note) in result.all_notes.iter().enumerate() { - let position = aligned_start + i as u64; - if let Some((note, _addr)) = try_decrypt_note(&views.prepared_ivk, raw_note) { - let cmx_bytes: [u8; 32] = match raw_note.cmx.as_slice().try_into() { - Ok(b) => b, - Err(_) => continue, - }; - decrypted_by_subwallet - .entry(*id) - .or_default() - .push(DiscoveredNote { - position, - cmx: cmx_bytes, - note, - }); - } - } - } + let stream = sync_shielded_notes_stream(sdk, &driver_ivk, aligned_start, Some(sync_config)); + futures::pin_mut!(stream); - let mut store = store.write().await; + // Route decryptions to the subwallet that owns the IVK, accumulated + // across every batch. + let mut decrypted_by_subwallet: BTreeMap> = BTreeMap::new(); + // Cumulative tree-append bookkeeping, interleaved with the fetch. + // // Append every commitment to the shared tree exactly once per // position, ALWAYS retained (`marked = true`). Skip positions // already in the tree (`global_pos < tree_size`) — the SDK // re-fetches from a chunk boundary every pass while the buffer // chunk is mutable, and a lagging subwallet rewinds that start - // even further, so `all_notes` routinely overlaps positions the - // tree already holds. Gating on the tree's own leaf count (NOT - // a per-subwallet watermark) is what makes the append - // idempotent: re-appending an existing position duplicates a - // leaf, corrupts shardtree's internal nodes, and makes + // even further, so the streamed notes routinely overlap positions + // the tree already holds. Gating on the tree's snapshot leaf count + // captured BEFORE the stream (NOT a per-subwallet watermark, and + // NOT the live tree size which we are actively growing) is what + // makes the append idempotent: re-appending an existing position + // duplicates a leaf, corrupts shardtree's internal nodes, and makes // per-position witnesses resolve against roots Platform never // recorded ("Anchor not found in the recorded anchors tree"). // @@ -317,19 +340,122 @@ pub(super) async fn sync_notes_across( // optimization can prune auth paths for positions no live // subwallet owns once all wallets have caught past them. let mut appended = 0u32; - for (i, raw_note) in result.all_notes.iter().enumerate() { - let global_pos = aligned_start + i as u64; - if global_pos < tree_size { - continue; + // Cumulative leaves committed to the tree this pass — the + // tree-progress ("checked") signal numerator. Includes only + // positions we actually appended (gate-skipped re-fetched + // positions are already in the tree, so they don't advance the + // commit count beyond `tree_size`). + let mut leaves_committed: u64 = tree_size; + // Cumulative notes scanned across every batch (decrypted + skipped) + // — drives the watermark advance and the host-visible scan volume, + // exactly as `result.total_notes_scanned` did in the one-shot path. + let mut total_notes_scanned: u64 = 0; + // Max block height across batches and the resume point for + // `next_start_index` semantics, accumulated as batches arrive. + let mut max_block_height: u64 = 0; + // Mirrors the one-shot rewind rule: track the last non-empty + // batch's `(start_index, is_partial)` so we can warn on a + // rewind-to-zero just like before. + let mut last_nonempty: Option<(u64, bool)> = None; + + while let Some(item) = stream.next().await { + let batch = item.map_err(|e| PlatformWalletError::ShieldedSyncFailed(e.to_string()))?; + max_block_height = max_block_height.max(batch.block_height); + // The denominator arrives with the batch (extracted from the + // note-fetch proof). It is stable across a sync; take the max-seen + // so a late chunk proven at a slightly higher block never lowers + // it. Stays 0 (indeterminate) only if no batch is ever produced. + total_target = total_target.max(batch.total_count); + total_notes_scanned += batch.notes.len() as u64; + if !batch.notes.is_empty() { + last_nonempty = Some((batch.start_index, batch.is_partial)); } - let cmx_bytes: [u8; 32] = - raw_note.cmx.as_slice().try_into().map_err(|_| { + + // 1. Append commitments for THIS batch, applying the same + // idempotency gate against the pre-stream `tree_size` + // snapshot. `batch.start_index + i` is the global tree + // position. + for (i, raw_note) in batch.notes.iter().enumerate() { + let global_pos = batch.start_index + i as u64; + if global_pos < tree_size { + continue; + } + let cmx_bytes: [u8; 32] = raw_note.cmx.as_slice().try_into().map_err(|_| { PlatformWalletError::ShieldedSyncFailed("Invalid cmx length".into()) })?; - store - .append_commitment(&cmx_bytes, true) - .map_err(|e| PlatformWalletError::ShieldedTreeUpdateFailed(e.to_string()))?; - appended += 1; + store + .append_commitment(&cmx_bytes, true) + .map_err(|e| PlatformWalletError::ShieldedTreeUpdateFailed(e.to_string()))?; + appended += 1; + leaves_committed += 1; + } + + // 2. Fire the tree-progress ("checked") callback once per batch + // (already coarse at ~8192-note batches). `total_target` is + // sourced from this batch's proof above; it is set by the first + // batch and stable thereafter. It is 0 (indeterminate on the + // host) only before any batch lands — which can't happen inside + // this loop since we're holding a batch. + if let Some(cb) = on_tree_progress { + cb(leaves_committed, total_target); + } + + // 3. Trial-decrypt THIS batch. Driver hits come pre-decrypted + // in `batch.decrypted`; other subwallets via local + // trial-decryption over `batch.notes`. + for dn in batch.decrypted { + decrypted_by_subwallet + .entry(*driver_id) + .or_default() + .push(DiscoveredNote { + position: dn.position, + cmx: dn.cmx, + note: dn.note, + }); + } + for (id, views) in subwallets.iter().skip(1) { + for (i, raw_note) in batch.notes.iter().enumerate() { + let position = batch.start_index + i as u64; + if let Some((note, _addr)) = try_decrypt_note(&views.prepared_ivk, raw_note) { + let cmx_bytes: [u8; 32] = match raw_note.cmx.as_slice().try_into() { + Ok(b) => b, + Err(_) => continue, + }; + decrypted_by_subwallet + .entry(*id) + .or_default() + .push(DiscoveredNote { + position, + cmx: cmx_bytes, + note, + }); + } + } + } + } + + // Preserve the one-shot `next_start_index` warning: if the resume + // point would rewind to 0 after scanning notes, the next sync + // rescans from the beginning. + let next_start_index = match last_nonempty { + Some((s, true)) => s, + _ => aligned_start + total_notes_scanned, + }; + info!( + total_scanned = total_notes_scanned, + decrypted_for_driver = decrypted_by_subwallet + .get(driver_id) + .map(|v| v.len()) + .unwrap_or(0), + next_start_index, + "SDK stream consumed" + ); + if next_start_index == 0 && total_notes_scanned > 0 { + warn!( + "Shielded sync: next_start_index is 0 after scanning {} notes — \ + next sync will rescan from the beginning", + total_notes_scanned, + ); } if appended > 0 { @@ -401,7 +527,7 @@ pub(super) async fn sync_notes_across( position: d.position, cmx: d.cmx, nullifier: nullifier.to_bytes(), - block_height: result.block_height, + block_height: max_block_height, is_spent: false, value, }; @@ -415,8 +541,10 @@ pub(super) async fn sync_notes_across( // Advance every subwallet's watermark to the same global // tree position so the next sync resumes coherently across - // the union. - let new_index = aligned_start + result.total_notes_scanned; + // the union. `total_notes_scanned` is accumulated from every + // streamed batch's note count — identical to the one-shot path's + // `result.total_notes_scanned`. + let new_index = aligned_start + total_notes_scanned; for (id, _) in subwallets { store .set_last_synced_note_index(*id, new_index) @@ -441,7 +569,7 @@ pub(super) async fn sync_notes_across( // already holds (chunk-boundary realignment, lagging-subwallet // rewind), and the host counter is documented as scan throughput, // not tree growth. - let scanned_volume = (aligned_start + result.total_notes_scanned).saturating_sub(already_have); + let scanned_volume = (aligned_start + total_notes_scanned).saturating_sub(already_have); Ok(MultiSyncNotesResult { per_subwallet_new_notes, total_scanned: scanned_volume, diff --git a/packages/rs-platform-wallet/tests/shielded_decrypt_bench.rs b/packages/rs-platform-wallet/tests/shielded_decrypt_bench.rs new file mode 100644 index 00000000000..0e930f3aebc --- /dev/null +++ b/packages/rs-platform-wallet/tests/shielded_decrypt_bench.rs @@ -0,0 +1,132 @@ +//! Microbenchmark for `dash_sdk::platform::shielded::try_decrypt_note`. +//! +//! Run: +//! cargo test -p platform-wallet --release --features shielded \ +//! shielded_decrypt_bench -- --ignored --nocapture +//! +//! Why this benchmark exists: a cold sync of the SDK_TEST_DATA chain (1M +//! filler notes) takes ~3 minutes on iPhone with ~50% CPU. The SDK's +//! sequential `for chunk in chunks { for note in chunk { try_decrypt_note(...) +//! } }` loop runs single-threaded after all chunks are fetched. This bench +//! isolates the CPU-bound decrypt path from network I/O so we can measure +//! single-threaded vs rayon-parallel throughput independently of DAPI +//! roundtrips. +//! +//! Note generation mirrors the on-chain SDK_TEST_DATA layout: random `rho`, +//! random encrypted payload of the canonical 216-byte wire length. The decrypt +//! path's early-out for non-Pallas `nullifier` bytes (~1/4 of random 32-byte +//! values) makes filler notes cheap to reject — but on a real chain every +//! recorded nullifier IS a valid Pallas element, so the realistic per-note +//! cost is somewhat higher. Treat these numbers as a lower bound on +//! production-chain decrypt time. + +#![cfg(feature = "shielded")] + +use std::time::Instant; + +use dash_sdk::platform::shielded::try_decrypt_note; +use dashcore::Network; +use drive_proof_verifier::types::ShieldedEncryptedNote; +use rand::{rngs::StdRng, RngCore, SeedableRng}; +use rayon::prelude::*; + +use platform_wallet::wallet::shielded::keys::OrchardKeySet; + +const ENCRYPTED_NOTE_WIRE_LEN: usize = 216; +const SEED_BENCH: [u8; 32] = [0x73; 32]; // matches SEED_A in drive-abci's seeder + +/// Generate `count` filler `ShieldedEncryptedNote`s with random bytes +/// matching the on-chain wire layout. Deterministic given `rng_seed`. +fn generate_filler_notes(count: usize, rng_seed: u64) -> Vec { + let mut rng = StdRng::seed_from_u64(rng_seed); + let mut out = Vec::with_capacity(count); + for _ in 0..count { + let mut cmx = vec![0u8; 32]; + let mut nullifier = vec![0u8; 32]; + let mut encrypted_note = vec![0u8; ENCRYPTED_NOTE_WIRE_LEN]; + rng.fill_bytes(&mut cmx); + rng.fill_bytes(&mut nullifier); + rng.fill_bytes(&mut encrypted_note); + out.push(ShieldedEncryptedNote { + cmx, + nullifier, + encrypted_note, + }); + } + out +} + +#[test] +#[ignore = "Microbenchmark; opt in via --ignored --nocapture"] +fn shielded_decrypt_bench() { + // 100k is enough for stable rates while staying ~1-10s per run on dev + // hardware. Bumping to 1M gives wall-clock that matches the iPhone cold + // sync; useful for cross-check but iterates slowly. + let count: usize = std::env::var("BENCH_COUNT") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(100_000); + + println!("== Shielded decrypt benchmark =="); + println!("Note count: {}", count); + println!("Rayon threads: {}", rayon::current_num_threads()); + + let key_set = + OrchardKeySet::from_seed(&SEED_BENCH, Network::Regtest, 0).expect("derive Orchard keys"); + let prepared_ivk = + grovedb_commitment_tree::PreparedIncomingViewingKey::new(&key_set.incoming_viewing_key); + + print!("Generating {} notes...", count); + let gen_start = Instant::now(); + let notes = generate_filler_notes(count, 0xDEAD_BEEF); + let gen_ms = gen_start.elapsed().as_millis(); + println!(" {} ms", gen_ms); + + // Warm-up: run a small batch to amortize any first-call setup. + for n in notes.iter().take(1024) { + let _ = try_decrypt_note(&prepared_ivk, n); + } + + // --- Single-threaded --- + let st_start = Instant::now(); + let mut st_hits = 0usize; + for n in ¬es { + if try_decrypt_note(&prepared_ivk, n).is_some() { + st_hits += 1; + } + } + let st_elapsed = st_start.elapsed(); + let st_rate = count as f64 / st_elapsed.as_secs_f64(); + println!( + "Single-threaded: {:>8.2} ms total, {:>9.0} notes/sec, hits={}", + st_elapsed.as_secs_f64() * 1000.0, + st_rate, + st_hits + ); + + // --- Rayon parallel --- + let par_start = Instant::now(); + let par_hits: usize = notes + .par_iter() + .filter(|n| try_decrypt_note(&prepared_ivk, n).is_some()) + .count(); + let par_elapsed = par_start.elapsed(); + let par_rate = count as f64 / par_elapsed.as_secs_f64(); + println!( + "Rayon parallel: {:>8.2} ms total, {:>9.0} notes/sec, hits={}", + par_elapsed.as_secs_f64() * 1000.0, + par_rate, + par_hits + ); + + let speedup = par_rate / st_rate; + println!("Speedup: {:.2}×", speedup); + + // Project to 1M-note cold sync wall-clock for the decrypt phase only + // (fetch / append / save are separate). + let proj_st_s = 1_000_000.0 / st_rate; + let proj_par_s = 1_000_000.0 / par_rate; + println!("Projected for 1M notes:"); + println!(" Single-threaded: {:>6.1} s", proj_st_s); + println!(" Rayon parallel: {:>6.1} s", proj_par_s); +} diff --git a/packages/rs-platform-wallet/tests/shielded_tree_append_bench.rs b/packages/rs-platform-wallet/tests/shielded_tree_append_bench.rs new file mode 100644 index 00000000000..d77eaa1890b --- /dev/null +++ b/packages/rs-platform-wallet/tests/shielded_tree_append_bench.rs @@ -0,0 +1,170 @@ +//! Microbenchmark for client-side commitment tree append throughput. +//! +//! The wallet's `ClientPersistentCommitmentTree` (SQLite-backed shardtree +//! over Orchard's `MerkleHashOrchard` / Sinsemilla) is what every cmx the +//! SDK fetches gets appended into during sync. I claimed earlier that this +//! is cheap because shardtree defers internal hash computation, but never +//! actually measured it. This bench settles whether tree append is a real +//! component of cold-sync wall-clock. +//! +//! Generates N random cmx values, opens a fresh tree at a temp SQLite path, +//! and times the `append(cmx, Retention::Marked)` loop end-to-end. `Marked` +//! is what the wallet uses (mark every position so any later-bound wallet +//! can retrieve a witness — see `packages/rs-platform-wallet/src/wallet/ +//! shielded/sync.rs:286-319`). +//! +//! Run: +//! cargo test -p platform-wallet --release --features shielded \ +//! --test shielded_tree_append_bench -- --ignored --nocapture +//! +//! Env overrides: +//! - `BENCH_COUNT` number of cmx to insert (default 100000) + +#![cfg(feature = "shielded")] + +use std::time::Instant; + +use grovedb_commitment_tree::{ClientPersistentCommitmentTree, Retention}; +use rand::{rngs::StdRng, RngCore, SeedableRng}; +use rusqlite::Connection; +use std::sync::{Arc, Mutex}; + +#[test] +#[ignore = "Microbenchmark; opt in via --ignored --nocapture"] +fn shielded_tree_append_bench() { + let count: usize = std::env::var("BENCH_COUNT") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(100_000); + + println!("== Client commitment tree append benchmark =="); + println!("Insertions: {}", count); + + // Fresh SQLite at a unique temp path each run so we measure cold tree + // build, not incremental append on top of existing rows. + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = std::env::temp_dir().join(format!("tree_append_bench_{}.sqlite", nanos)); + println!("Tree path: {}", path.display()); + + // Generate cmx bytes up front so the inner loop doesn't include RNG cost. + print!("Generating {} cmx values...", count); + let gen_start = Instant::now(); + let mut rng = StdRng::seed_from_u64(0xCAFEF00D); + let mut leaves: Vec<[u8; 32]> = Vec::with_capacity(count); + for _ in 0..count { + // Most random 32-byte values are valid Pallas field elements; + // `merkle_hash_from_bytes` (called inside append) parses bytes as + // Pallas. To get a clean measurement, we skip any that fail to parse + // — in practice ~3/4 of random bytes are valid so this loop is fast. + loop { + let mut bytes = [0u8; 32]; + rng.fill_bytes(&mut bytes); + if grovedb_commitment_tree::merkle_hash_from_bytes(&bytes).is_some() { + leaves.push(bytes); + break; + } + } + } + let gen_ms = gen_start.elapsed().as_millis(); + println!(" {} ms", gen_ms); + + // Run THREE variants on the same workload to attribute cost: + // 1) Default settings (what the wallet currently uses) — DELETE journal, + // synchronous=FULL: fsync per commit. + // 2) WAL + synchronous=NORMAL — typical app-grade tuning, batches fsyncs. + // 3) In-memory tree — eliminates I/O entirely, isolating pure shardtree + // + Sinsemilla CPU cost. + fn bench_one(label: &str, leaves: &[[u8; 32]], setup: impl FnOnce() -> Connection) { + let conn = setup(); + let shared = Arc::new(Mutex::new(conn)); + let mut tree = ClientPersistentCommitmentTree::open_on_shared_connection(shared, 100) + .expect("open commitment tree"); + let start = Instant::now(); + for cmx in leaves { + tree.append(*cmx, Retention::Marked) + .expect("append commitment"); + } + let elapsed = start.elapsed(); + let per_us = elapsed.as_micros() as f64 / leaves.len() as f64; + let rate = leaves.len() as f64 / elapsed.as_secs_f64(); + println!( + "{:32} {:>9.2} s {:>9.1} µs/append {:>9.0} appends/sec (proj 1M: {:.1} s)", + label, + elapsed.as_secs_f64(), + per_us, + rate, + 1_000_000.0 / rate + ); + } + + println!(); + println!( + "{:<32} {:>11} {:>17} {:>21} {:>17}", + "config", "total", "per-append", "throughput", "projected 1M" + ); + println!("{}", "-".repeat(110)); + + // Variant 1: default (what the wallet does today) + let p1 = path.clone(); + bench_one("default (DELETE + sync=FULL)", &leaves, || { + Connection::open(&p1).expect("open sqlite default") + }); + let _ = std::fs::remove_file(&path); + + // Variant 2: WAL + sync=NORMAL + let p2 = path.clone(); + bench_one("WAL + sync=NORMAL", &leaves, || { + let c = Connection::open(&p2).expect("open sqlite wal"); + c.pragma_update(None, "journal_mode", "WAL").expect("WAL"); + c.pragma_update(None, "synchronous", "NORMAL") + .expect("sync"); + c.pragma_update(None, "temp_store", "MEMORY") + .expect("temp_store"); + c + }); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(format!("{}-wal", path.display())); + let _ = std::fs::remove_file(format!("{}-shm", path.display())); + + // Variant 3: in-memory (CPU only, no I/O) + bench_one(":memory: (CPU only)", &leaves, || { + Connection::open_in_memory().expect("open in-memory") + }); + + // Variant 4: the wallet's actual path via `FileBackedShieldedStore::open_path`. + // This is the path the wallet hits during real cold sync — verifies the + // WAL + NORMAL PRAGMA fix in `file_store.rs` lands in production code. + { + use platform_wallet::wallet::shielded::file_store::FileBackedShieldedStore; + use platform_wallet::wallet::shielded::store::ShieldedStore; + let p4 = path.clone(); + let mut store = + FileBackedShieldedStore::open_path(&p4, 100).expect("open FileBackedShieldedStore"); + let start = Instant::now(); + for cmx in &leaves { + store + .append_commitment(cmx, true) + .expect("append_commitment"); + } + let elapsed = start.elapsed(); + let per_us = elapsed.as_micros() as f64 / count as f64; + let rate = count as f64 / elapsed.as_secs_f64(); + println!( + "{:32} {:>9.2} s {:>9.1} µs/append {:>9.0} appends/sec (proj 1M: {:.1} s)", + "FileBackedShieldedStore (wallet)", + elapsed.as_secs_f64(), + per_us, + rate, + 1_000_000.0 / rate + ); + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(format!("{}-wal", path.display())); + let _ = std::fs::remove_file(format!("{}-shm", path.display())); + } + + // `max_leaf_position` reporting omitted — the per-variant table is the + // useful output; final tree state is identical across variants. +} diff --git a/packages/rs-sdk/Cargo.toml b/packages/rs-sdk/Cargo.toml index 2937843e192..d918fee30ef 100644 --- a/packages/rs-sdk/Cargo.toml +++ b/packages/rs-sdk/Cargo.toml @@ -18,7 +18,7 @@ drive = { path = "../rs-drive", default-features = false, features = [ ] } drive-proof-verifier = { path = "../rs-drive-proof-verifier", default-features = false } -grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "60f29685172653f6007e63d0916bce4633bc23b9", features = [ +grovedb-commitment-tree = { git = "https://github.com/dashpay/grovedb", rev = "5eb7a5380a6e974513343352acfd6b30a8c1f87c", features = [ "client", "sqlite", ], optional = true } diff --git a/packages/rs-sdk/README.md b/packages/rs-sdk/README.md index 7a2372e0d4e..9a33a75b7b2 100644 --- a/packages/rs-sdk/README.md +++ b/packages/rs-sdk/README.md @@ -85,7 +85,7 @@ Refer to rich comments / help in the forementioned scripts for more details. ### SDK test data -When starting the local devnet with `SDK_TEST_DATA=true yarn start`, the `create_sdk_test_data` cfg flag +When starting the local devnet with `yarn start` (the `local` dashmate config has `buildArgs.SDK_TEST_DATA = "true"` set by `yarn setup` — see `scripts/configure_dashmate.sh`), the `create_sdk_test_data` cfg flag activates creation of deterministic test data in genesis state. This data is defined in `packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/`. @@ -109,7 +109,7 @@ The `.env` file is automatically generated during `yarn setup` or `yarn reset`, To generate test vectors: -1. Start local dev environment of Dash Platform using `SDK_TEST_DATA=true yarn start`. +1. Start local dev environment of Dash Platform using `yarn start` (the `local` dashmate config has `buildArgs.SDK_TEST_DATA = "true"` set by `yarn setup` — see `scripts/configure_dashmate.sh`). 2. Ensure platform address and credentials in `packages/rs-sdk/tests/.env` are correct. 3. Run `packages/rs-sdk/scripts/generate_test_vectors.sh` script. 4. (Optional) commit generated files with `git commit packages/rs-sdk/tests/vectors/`. @@ -168,7 +168,7 @@ in `packages/rs-dapi-client/src/transport/grpc.rs`. 12. [ ] (Optional) If not already configured, run `yarn setup` (fresh checkout) or `yarn reset` (reconfigure existing environment). **Warning:** both commands rebuild everything and reset data — do not run if your environment is already working. This configures the `.env` file in `packages/rs-sdk/tests/` needed by the tests. -13. [ ] Start local devnet with `SDK_TEST_DATA=true yarn start`. +13. [ ] Start local devnet with `yarn start` (the `local` dashmate config has `buildArgs.SDK_TEST_DATA = "true"` set by `yarn setup` — see `scripts/configure_dashmate.sh`). 14. [ ] Generate test vectors by running `packages/rs-sdk/scripts/generate_test_vectors.sh test_name` where `test_name` matches only the new tests (e.g., `test_token_pre_programmed_distributions`). Running without arguments regenerates **all** vectors — avoid this unless intentional. diff --git a/packages/rs-sdk/src/platform/shielded/mod.rs b/packages/rs-sdk/src/platform/shielded/mod.rs index 117a6ad9210..c578a2b402e 100644 --- a/packages/rs-sdk/src/platform/shielded/mod.rs +++ b/packages/rs-sdk/src/platform/shielded/mod.rs @@ -9,5 +9,7 @@ pub mod notes_sync; pub mod nullifier_sync; pub use notes_sync::decrypt::try_decrypt_note; -pub use notes_sync::sync_shielded_notes::sync_shielded_notes; -pub use notes_sync::types::{DecryptedNote, ShieldedSyncConfig, ShieldedSyncResult}; +pub use notes_sync::sync_shielded_notes::{sync_shielded_notes, sync_shielded_notes_stream}; +pub use notes_sync::types::{ + DecryptedNote, ShieldedChunkBatch, ShieldedSyncConfig, ShieldedSyncResult, +}; diff --git a/packages/rs-sdk/src/platform/shielded/notes_sync/fetch_chunk.rs b/packages/rs-sdk/src/platform/shielded/notes_sync/fetch_chunk.rs index 10484084e45..68afdce0287 100644 --- a/packages/rs-sdk/src/platform/shielded/notes_sync/fetch_chunk.rs +++ b/packages/rs-sdk/src/platform/shielded/notes_sync/fetch_chunk.rs @@ -8,14 +8,21 @@ use tracing::debug; /// Fetch a single chunk of encrypted notes from the network. /// -/// Returns `(chunk_start_index, notes, block_height)`. An empty vec means no -/// notes exist at this position (past end of tree). +/// Returns `(chunk_start_index, notes, block_height, total_count)`. An empty +/// vec means no notes exist at this position (past end of tree). +/// +/// `total_count` is the on-chain total number of notes in the shielded +/// `CommitmentTree`, extracted from the SAME note-fetch proof (the parent +/// CommitmentTree element is always present in it). It is stable across a +/// sync and arrives on every chunk — including the empty final chunk — so a +/// wallet gets the sync progress-bar denominator for free with no separate +/// `GetShieldedNotesCount` RPC. pub async fn fetch_chunk( sdk: &Sdk, chunk_start: u64, chunk_size: u64, settings: RequestSettings, -) -> Result<(u64, Vec, u64), Error> { +) -> Result<(u64, Vec, u64, u64), Error> { let query = ShieldedEncryptedNotesQuery { start_index: chunk_start, count: chunk_size as u32, @@ -26,17 +33,18 @@ pub async fn fetch_chunk( let (result, metadata) = ShieldedEncryptedNotes::fetch_with_metadata(sdk, query, Some(settings)).await?; - let notes = match result { - Some(ShieldedEncryptedNotes(notes)) => notes, - None => Vec::new(), + let (notes, total_count) = match result { + Some(ShieldedEncryptedNotes { notes, total_count }) => (notes, total_count), + None => (Vec::new(), 0), }; debug!( chunk_start, notes_returned = notes.len(), block_height = metadata.height, + total_count, "shielded notes chunk fetched" ); - Ok((chunk_start, notes, metadata.height)) + Ok((chunk_start, notes, metadata.height, total_count)) } diff --git a/packages/rs-sdk/src/platform/shielded/notes_sync/sync_shielded_notes.rs b/packages/rs-sdk/src/platform/shielded/notes_sync/sync_shielded_notes.rs index 67544a2f29e..b19081df1cc 100644 --- a/packages/rs-sdk/src/platform/shielded/notes_sync/sync_shielded_notes.rs +++ b/packages/rs-sdk/src/platform/shielded/notes_sync/sync_shielded_notes.rs @@ -1,51 +1,24 @@ use super::decrypt::try_decrypt_note; use super::fetch_chunk::fetch_chunk as do_fetch_chunk; -use super::types::{DecryptedNote, ShieldedSyncConfig, ShieldedSyncResult}; +use super::types::{DecryptedNote, ShieldedChunkBatch, ShieldedSyncConfig, ShieldedSyncResult}; use crate::{Error, Sdk}; use drive_proof_verifier::types::ShieldedEncryptedNote; -use futures::stream::{FuturesUnordered, StreamExt}; +use futures::stream::{FuturesUnordered, Stream, StreamExt}; use grovedb_commitment_tree::PreparedIncomingViewingKey; use std::collections::BTreeMap; use std::future::Future; use std::pin::Pin; use tracing::debug; -/// Fetch all shielded encrypted notes starting from `start_index`, query -/// multiple nodes in parallel, and perform trial decryption. -/// -/// This is the main entry point for wallet sync. It handles: -/// 1. Chunk-aligned pagination (each query covers one BulkAppendTree chunk) -/// 2. Parallel dispatch of chunk queries across network nodes -/// 3. Proof verification on every response -/// 4. Trial decryption with the provided incoming viewing key +/// Resolve the on-chain MMR chunk size and the per-request fetch size. /// -/// # Arguments -/// -/// - `sdk` — SDK instance connected to the network -/// - `ivk` — prepared incoming viewing key for trial decryption -/// - `start_index` — first note position to fetch (must be a multiple of -/// the chunk size, typically 2048) -/// - `config` — optional tuning; `None` uses sensible defaults -/// -/// # Returns -/// -/// [`ShieldedSyncResult`] containing decrypted notes that belong to the -/// viewer, all raw notes for commitment tree updates, and the next index -/// to resume from. -pub async fn sync_shielded_notes( - sdk: &Sdk, - ivk: &PreparedIncomingViewingKey, - start_index: u64, - config: Option, -) -> Result { - let config = config.unwrap_or_default(); - - // `mmr_chunk_size` is the on-chain MMR chunk size — the unit that - // `start_index` must align to. `fetch_size` is how many notes we - // pull per request; under `max_query_chunks=4` the server packs 4 - // MMR chunks into one proof, so each request advances by 4× the - // MMR chunk size. Decoupling the two means the SDK can opportunistically - // request larger spans without touching the on-chain tree shape. +/// `mmr_chunk_size` is the unit that `start_index` must align to. +/// `fetch_size` is how many notes we pull per request: under +/// `max_query_chunks` the server packs that many MMR chunks into one +/// proof, so each request advances by `max_query_chunks ×` the MMR +/// chunk size. Decoupling the two lets the SDK opportunistically +/// request larger spans without touching the on-chain tree shape. +fn resolve_sizes(sdk: &Sdk) -> (u64, u64) { let mmr_chunk_size: u64 = 1u64 << drive::drive::shielded::paths::SHIELDED_NOTES_CHUNK_POWER; let max_query_chunks = sdk .version() @@ -56,119 +29,386 @@ pub async fn sync_shielded_notes( let fetch_size = mmr_chunk_size .saturating_mul(max_query_chunks) .max(mmr_chunk_size); + (mmr_chunk_size, fetch_size) +} - // Validate alignment against the MMR chunk size (NOT the multi-chunk - // fetch size). The server only requires per-MMR-chunk alignment; any - // multiple of `mmr_chunk_size` is a legal start. A `start_index` - // produced by a previous sync pass will be a multiple of `fetch_size` - // (and therefore of `mmr_chunk_size`) so this check is normally a - // no-op — but a hand-built resume point could land on a non-fetch - // boundary and still be valid. - if mmr_chunk_size > 0 && !start_index.is_multiple_of(mmr_chunk_size) { - return Err(Error::Generic(format!( - "start_index {} is not chunk-aligned; must be a multiple of {}", - start_index, mmr_chunk_size - ))); - } - - let chunk_size = fetch_size; - - let max_concurrent = config.max_concurrent.max(1); - let settings = config.request_settings; - - type ChunkFuture = - Pin, u64), Error>> + Send>>; +type ChunkFuture = Pin< + Box, u64, u64), Error>> + Send>, +>; - // Sliding-window parallel fetch using FuturesUnordered. - // Each future fetches one chunk and returns (chunk_start_index, notes, block_height). - let mut futures: FuturesUnordered = FuturesUnordered::new(); - let mut next_chunk_index = start_index; - let mut reached_end = false; +/// Pure, network-free reorder buffer + emit watermark. +/// +/// Chunk fetches complete out of order under the sliding-window +/// `FuturesUnordered`, but tree-position consumers require strictly +/// ascending `start_index`. This buffer holds early-finishing later +/// chunks until every predecessor has been emitted. The watermark +/// advances by `chunk_size` on each successful pop, so a chunk at +/// `start_index = watermark` can only emit once its predecessor at +/// `watermark - chunk_size` already has. +/// +/// Generic over the buffered payload `T` so the ordering logic can be +/// unit-tested in isolation (no `Sdk`, no trial-decryption). +struct ReorderBuffer { + chunk_size: u64, + /// Completed chunks waiting on a predecessor, keyed by start index. + buffered: BTreeMap, + /// Next start index allowed to emit (tree order). + watermark: u64, +} - // Seed initial batch of chunk queries - for _ in 0..max_concurrent { - let chunk_idx = next_chunk_index; - next_chunk_index += chunk_size; - let sdk = sdk.clone(); - futures.push(Box::pin(async move { - do_fetch_chunk(&sdk, chunk_idx, chunk_size, settings).await - })); +impl ReorderBuffer { + fn new(start_index: u64, chunk_size: u64) -> Self { + Self { + chunk_size, + buffered: BTreeMap::new(), + watermark: start_index, + } } - // Collect results keyed by chunk start_index for ordered reassembly - let mut chunk_results: BTreeMap> = BTreeMap::new(); - let mut max_block_height: u64 = 0; + /// Buffer a completed chunk by its start index. + fn insert(&mut self, chunk_idx: u64, payload: T) { + self.buffered.insert(chunk_idx, payload); + } - while let Some(result) = futures.next().await { - let (chunk_idx, notes, block_height) = result?; - let is_partial = (notes.len() as u64) < chunk_size; - chunk_results.insert(chunk_idx, notes); - max_block_height = max_block_height.max(block_height); + /// If the chunk at the current watermark is buffered, remove it, + /// advance the watermark by `chunk_size`, and return + /// `(start_index, payload)`. Otherwise `None`. + fn pop_ready(&mut self) -> Option<(u64, T)> { + let start_index = self.watermark; + let payload = self.buffered.remove(&start_index)?; + self.watermark += self.chunk_size; + Some((start_index, payload)) + } +} - if is_partial { - reached_end = true; - } +/// Mutable driver state for the streaming chunk pipeline. +/// +/// Owned by the [`futures::stream::unfold`] closure inside +/// [`sync_shielded_notes_stream`]. Because the stream is pull-based, +/// the `FuturesUnordered` only advances when the consumer polls for the +/// next batch — that is the backpressure: a slow consumer (e.g. a +/// wallet tree-append) simply doesn't poll, so no further chunk fetch +/// is queued and in-flight network requests stay capped at +/// `max_concurrent`. The reorder buffer is bounded by the same window. +struct StreamState { + sdk: Sdk, + ivk: PreparedIncomingViewingKey, + chunk_size: u64, + /// The sync's absolute start position. Added to `cumulative_scanned` + /// when firing the download progress callback so the "Downloaded" + /// bar is absolute-toward-total — it shares a baseline with the + /// wallet's "Checked" (= tree leaf count) signal, keeping + /// Checked <= Downloaded even on a resume where the tree is full. + start_index: u64, + settings: rs_dapi_client::RequestSettings, + on_progress: Option, + /// In-flight chunk fetches (sliding window of `max_concurrent`). + futures: FuturesUnordered, + /// Next chunk start index to dispatch a fetch for. + next_chunk_index: u64, + /// Out-of-order completed chunks waiting for their predecessor to be + /// emitted, plus the emit watermark. Drained in ascending order. + /// Payload: `(notes, block_height)`. The progress denominator is NOT + /// buffered per chunk — the emitted batch carries `self.total_count` + /// (the monotonic max-seen) so out-of-order completion can never make + /// the determinate progress bar regress mid-sync. + reorder: ReorderBuffer<(Vec, u64)>, + /// Set once a partial (short) chunk is observed — stops queuing new + /// fetches. + reached_end: bool, + /// Cumulative notes seen across every completed chunk — fed into the + /// "downloaded" progress callback. + cumulative_scanned: u64, + /// Max block height across every completed chunk so far. + max_block_height: u64, + /// On-chain total note count extracted from the note-fetch proofs — + /// max/last-seen across completed chunks. Carried into every emitted + /// batch as the sync progress-bar denominator. + total_count: u64, +} - // Queue the next chunk if we haven't reached the end - if !reached_end { - let chunk_idx = next_chunk_index; - next_chunk_index += chunk_size; - let sdk = sdk.clone(); - futures.push(Box::pin(async move { - do_fetch_chunk(&sdk, chunk_idx, chunk_size, settings).await - })); +impl StreamState { + /// Queue one more chunk fetch if we haven't hit end-of-stream. + fn queue_next(&mut self) { + if self.reached_end { + return; } + let chunk_idx = self.next_chunk_index; + self.next_chunk_index += self.chunk_size; + let sdk = self.sdk.clone(); + let chunk_size = self.chunk_size; + let settings = self.settings; + self.futures.push(Box::pin(async move { + do_fetch_chunk(&sdk, chunk_idx, chunk_size, settings).await + })); } - // Flatten in tree order and perform trial decryption - let mut all_notes = Vec::new(); - let mut decrypted_notes = Vec::new(); + /// If the chunk at the emit watermark is buffered, build its batch, + /// advance the watermark, and return it. Otherwise `None`. + fn pop_ready(&mut self) -> Option { + let (start_index, (notes, block_height)) = self.reorder.pop_ready()?; + let is_partial = (notes.len() as u64) < self.chunk_size; - for (&chunk_start, notes) in &chunk_results { + // Trial-decrypt this chunk before emission (moved out of the + // post-loop the one-shot path used to run). + let mut decrypted = Vec::new(); for (i, note) in notes.iter().enumerate() { - let position = chunk_start + i as u64; - - if let Some((decrypted, address)) = try_decrypt_note(ivk, note) { + if let Some((dec, address)) = try_decrypt_note(&self.ivk, note) { let nf: [u8; 32] = note.nullifier.as_slice().try_into().unwrap_or([0u8; 32]); let cmx: [u8; 32] = note.cmx.as_slice().try_into().unwrap_or([0u8; 32]); - - decrypted_notes.push(DecryptedNote { - position, - note: decrypted, + decrypted.push(DecryptedNote { + position: start_index + i as u64, + note: dec, address, nullifier: nf, cmx, }); } } + + Some(ShieldedChunkBatch { + start_index, + notes, + decrypted, + block_height, + is_partial, + // Emit the monotonic max-seen denominator, not this chunk's + // own proven total. An older chunk completing after a newer + // proof already raised the count must NOT lower it again. + total_count: self.total_count, + }) } +} - let total_notes_scanned: u64 = chunk_results.values().map(|v| v.len() as u64).sum(); - - // Determine next_start_index before consuming chunk_results. - // If the last non-empty chunk is partial (fewer notes than chunk_size), - // resume from that chunk's start -- the BulkAppendTree buffer chunk is - // mutable and may receive more notes before the next sync. - let next_start_index = if chunk_size > 0 { - let last_partial = chunk_results - .iter() - .rev() - .find(|(_, notes)| !notes.is_empty()) - .filter(|(_, notes)| (notes.len() as u64) < chunk_size); - - match last_partial { - Some((&chunk_start, _)) => chunk_start, - None => start_index + total_notes_scanned, - } +/// Streaming variant of [`sync_shielded_notes`]: fetch shielded +/// encrypted notes starting at `start_index` with the existing +/// sliding-window parallelism, and yield each chunk **as soon as it is +/// the next contiguous one in tree order**. +/// +/// Out-of-order network completions wait in an internal reorder buffer +/// until their predecessor has emitted, so the returned stream is +/// guaranteed to produce [`ShieldedChunkBatch`]es in strictly ascending +/// `start_index` order. Trial decryption against `ivk` happens per +/// chunk just before emission. +/// +/// # Backpressure +/// +/// The stream is pull-based ([`futures::stream::unfold`]): the internal +/// `FuturesUnordered` only advances when the consumer polls for the +/// next item, and a new fetch is queued only after a batch is emitted. +/// A consumer slower than the network therefore caps in-flight fetches +/// at `max_concurrent` and bounds the reorder buffer to the same window +/// — memory stays bounded without a separate spawned producer task +/// (which would also be unavailable under `wasm32`). +/// +/// # Arguments +/// +/// - `sdk` — SDK instance connected to the network +/// - `ivk` — prepared incoming viewing key for trial decryption +/// - `start_index` — first note position to fetch (must be a multiple +/// of the MMR chunk size, typically 2048) +/// - `config` — optional tuning; `None` uses sensible defaults +/// +/// # Errors +/// +/// A non-chunk-aligned `start_index` yields a single `Err` item, after +/// which the stream is exhausted. A fetch error likewise surfaces as an +/// `Err` item; callers should stop consuming on the first error. +pub fn sync_shielded_notes_stream( + sdk: &Sdk, + ivk: &PreparedIncomingViewingKey, + start_index: u64, + config: Option, +) -> impl Stream> + Send { + let config = config.unwrap_or_default(); + let (mmr_chunk_size, fetch_size) = resolve_sizes(sdk); + + // Validate alignment against the MMR chunk size (NOT the multi-chunk + // fetch size). The server only requires per-MMR-chunk alignment; any + // multiple of `mmr_chunk_size` is a legal start. + let alignment_error = if mmr_chunk_size > 0 && !start_index.is_multiple_of(mmr_chunk_size) { + Some(Error::Generic(format!( + "start_index {} is not chunk-aligned; must be a multiple of {}", + start_index, mmr_chunk_size + ))) } else { - start_index + total_notes_scanned + None }; - // Move notes out of the BTreeMap in order - for (_, notes) in chunk_results { - all_notes.extend(notes); + let max_concurrent = config.max_concurrent.max(1); + let chunk_size = fetch_size; + + // Seed the initial sliding window of chunk queries. + let futures: FuturesUnordered = FuturesUnordered::new(); + let mut state = StreamState { + sdk: sdk.clone(), + ivk: ivk.clone(), + chunk_size, + start_index, + settings: config.request_settings, + on_progress: config.on_chunk_completed.clone(), + futures, + next_chunk_index: start_index, + reorder: ReorderBuffer::new(start_index, chunk_size), + reached_end: false, + cumulative_scanned: 0, + max_block_height: 0, + total_count: 0, + }; + for _ in 0..max_concurrent { + state.queue_next(); + } + + // `unfold` yields `Some((item, next_state))` to emit `item`, or + // `None` to end the stream. Each poll of the returned stream drives + // exactly enough of the `FuturesUnordered` to produce the next + // contiguous chunk, which is what makes backpressure pull-based. + futures::stream::unfold( + (state, alignment_error, false), + move |(mut state, mut alignment_error, done)| async move { + if done { + return None; + } + // Surface a pre-validated alignment error as the sole item. + if let Some(err) = alignment_error.take() { + return Some((Err(err), (state, None, true))); + } + + // A buffered chunk may already be ready (a predecessor just + // emitted on a prior poll). Emit it before touching the + // network again. + if let Some(batch) = state.pop_ready() { + // Queue replacement work for the chunk we just emitted so + // the sliding window stays full. + state.queue_next(); + return Some((Ok(batch), (state, None, false))); + } + + // Pull completed fetches until the watermark chunk is ready. + loop { + let next = state.futures.next().await; + let Some(result) = next else { + // No in-flight fetches and nothing buffered: stream + // is exhausted. + return None; + }; + let (chunk_idx, notes, block_height, total_count) = match result { + Ok(v) => v, + Err(e) => return Some((Err(e), (state, None, true))), + }; + + let is_partial = (notes.len() as u64) < state.chunk_size; + state.cumulative_scanned += notes.len() as u64; + state.max_block_height = state.max_block_height.max(block_height); + // The on-chain total is stable across a sync; take the + // max-seen so a late-arriving chunk proven at a slightly + // higher block never lowers the denominator. + state.total_count = state.total_count.max(total_count); + if is_partial { + state.reached_end = true; + } + // "Downloaded" progress fires per network chunk + // completion, preserving the existing meaning. + if let Some(cb) = state.on_progress.as_ref() { + // Absolute downloaded position (start_index + cumulative + // scanned). `start_index` can rewind below prior progress + // on a resume, so a consumer that also tracks committed + // ("checked") progress must clamp this to its own baseline + // to keep "Downloaded" from reading below "Checked" — the + // wallet does exactly that in `sync_notes_across`. + cb( + state.start_index + state.cumulative_scanned, + state.max_block_height, + ); + } + state.reorder.insert(chunk_idx, (notes, block_height)); + + if let Some(batch) = state.pop_ready() { + state.queue_next(); + return Some((Ok(batch), (state, None, false))); + } + // Out-of-order completion: keep draining in-flight + // fetches until the watermark chunk arrives. + } + }, + ) +} + +/// Fetch all shielded encrypted notes starting from `start_index`, query +/// multiple nodes in parallel, and perform trial decryption. +/// +/// This is the one-shot entry point for wallet sync. It drives +/// [`sync_shielded_notes_stream`] to completion and assembles a single +/// [`ShieldedSyncResult`]. It handles: +/// 1. Chunk-aligned pagination (each query covers one BulkAppendTree chunk) +/// 2. Parallel dispatch of chunk queries across network nodes +/// 3. Proof verification on every response +/// 4. Trial decryption with the provided incoming viewing key +/// +/// Prefer [`sync_shielded_notes_stream`] when the consumer can overlap +/// per-chunk work (e.g. tree-append) with later fetches. +/// +/// # Arguments +/// +/// - `sdk` — SDK instance connected to the network +/// - `ivk` — prepared incoming viewing key for trial decryption +/// - `start_index` — first note position to fetch (must be a multiple of +/// the chunk size, typically 2048) +/// - `config` — optional tuning; `None` uses sensible defaults +/// +/// # Returns +/// +/// [`ShieldedSyncResult`] containing decrypted notes that belong to the +/// viewer, all raw notes for commitment tree updates, and the next index +/// to resume from. +pub async fn sync_shielded_notes( + sdk: &Sdk, + ivk: &PreparedIncomingViewingKey, + start_index: u64, + config: Option, +) -> Result { + let stream = sync_shielded_notes_stream(sdk, ivk, start_index, config); + futures::pin_mut!(stream); + + let mut all_notes: Vec = Vec::new(); + let mut decrypted_notes: Vec = Vec::new(); + let mut total_notes_scanned: u64 = 0; + let mut max_block_height: u64 = 0; + // On-chain total note count from the note-fetch proofs (max-seen). The + // value is stable across a sync; max-seen guards against a late chunk + // proven at a slightly higher block lowering the denominator. + let mut total_count: u64 = 0; + // Mirrors the original one-shot logic exactly: track the LAST + // non-empty chunk's `(start_index, is_partial)`. Batches arrive in + // ascending `start_index`, so the last non-empty one we observe is + // the same chunk `chunk_results.iter().rev().find(non-empty)` would + // have selected. We only rewind `next_start_index` to it if that + // last non-empty chunk is itself partial (a short buffer chunk that + // may still grow before the next sync). Trailing empty chunks from + // the still-draining sliding window are ignored, just like the + // original `find(non-empty)` skipped them. + let mut last_nonempty: Option<(u64, bool)> = None; + + while let Some(item) = stream.next().await { + let batch = item?; + max_block_height = max_block_height.max(batch.block_height); + total_count = total_count.max(batch.total_count); + total_notes_scanned += batch.notes.len() as u64; + if !batch.notes.is_empty() { + last_nonempty = Some((batch.start_index, batch.is_partial)); + } + decrypted_notes.extend(batch.decrypted); + all_notes.extend(batch.notes); } + // Preserve `next_start_index` semantics exactly: if the last + // non-empty chunk is partial, rewind to its start; otherwise resume + // past everything scanned. + let next_start_index = match last_nonempty { + Some((s, true)) => s, + _ => start_index + total_notes_scanned, + }; + debug!( total_notes_scanned, decrypted_count = decrypted_notes.len(), @@ -182,5 +422,90 @@ pub async fn sync_shielded_notes( next_start_index, total_notes_scanned, block_height: max_block_height, + total_count, }) } + +#[cfg(test)] +mod tests { + use super::ReorderBuffer; + + /// The reorder buffer underpins the stream's core guarantee: + /// chunks emit in strictly ascending `start_index` even when the + /// underlying network fetches complete out of order. This drives + /// completions in a deliberately scrambled order and asserts the + /// emitted sequence is monotonically increasing and contiguous. + #[test] + fn reorder_emits_strictly_ascending_under_out_of_order_arrival() { + let chunk_size = 8192u64; + let start = 0u64; + let mut buf: ReorderBuffer = ReorderBuffer::new(start, chunk_size); + + // Five chunks at starts 0, 8192, 16384, 24576, 32768. Their + // network fetches complete in scrambled order; we feed that + // order in and pull whatever is ready after each arrival. + let starts: Vec = (0..5).map(|k| start + k * chunk_size).collect(); + // Arrival order: 2nd, 4th, 1st, 5th, 3rd (indices into `starts`). + let arrival = [1usize, 3, 0, 4, 2]; + + let mut emitted: Vec = Vec::new(); + for (payload, &idx) in arrival.iter().enumerate() { + buf.insert(starts[idx], payload); + // Drain everything that became contiguously ready. + while let Some((s, _payload)) = buf.pop_ready() { + emitted.push(s); + } + } + + // Every chunk emitted exactly once, in ascending tree order — + // i.e. identical to the in-order baseline regardless of the + // scrambled arrival order. + assert_eq!( + emitted, starts, + "reorder buffer must emit chunks in ascending start_index" + ); + // Strictly ascending check (defensive; redundant with the + // equality above but pins the invariant explicitly). + assert!( + emitted.windows(2).all(|w| w[0] < w[1]), + "emitted start_index sequence must be strictly ascending" + ); + } + + /// A later chunk that finishes first must NOT emit before its + /// predecessor — it waits in the buffer until the watermark reaches + /// it. Verifies the gate holds across interleaved insert/pop calls. + #[test] + fn reorder_holds_back_chunk_until_predecessor_emits() { + let chunk_size = 4u64; + let mut buf: ReorderBuffer<&'static str> = ReorderBuffer::new(0, chunk_size); + + // Chunk at start=4 (the second chunk) arrives first. + buf.insert(4, "second"); + // Nothing is emittable yet — watermark is still 0. + assert!(buf.pop_ready().is_none()); + + // Now the first chunk arrives. + buf.insert(0, "first"); + // Both should drain, in order. + assert_eq!(buf.pop_ready(), Some((0, "first"))); + assert_eq!(buf.pop_ready(), Some((4, "second"))); + assert!(buf.pop_ready().is_none()); + } + + /// Non-zero `start_index` (a resume point): the watermark must + /// begin at `start_index`, not 0, so a mid-tree resume emits its + /// first chunk immediately. + #[test] + fn reorder_respects_nonzero_start_index() { + let chunk_size = 8192u64; + let start = 16384u64; + let mut buf: ReorderBuffer = ReorderBuffer::new(start, chunk_size); + + buf.insert(start + chunk_size, 1); + assert!(buf.pop_ready().is_none(), "successor must wait"); + buf.insert(start, 0); + assert_eq!(buf.pop_ready(), Some((start, 0))); + assert_eq!(buf.pop_ready(), Some((start + chunk_size, 1))); + } +} diff --git a/packages/rs-sdk/src/platform/shielded/notes_sync/types.rs b/packages/rs-sdk/src/platform/shielded/notes_sync/types.rs index 75b3f8e167a..b25a280e8ee 100644 --- a/packages/rs-sdk/src/platform/shielded/notes_sync/types.rs +++ b/packages/rs-sdk/src/platform/shielded/notes_sync/types.rs @@ -3,16 +3,36 @@ use drive_proof_verifier::types::ShieldedEncryptedNote; use grovedb_commitment_tree::{Note, PaymentAddress}; use rs_dapi_client::RequestSettings; +use std::sync::Arc; /// Default maximum number of chunk queries in flight at once. pub const DEFAULT_MAX_CONCURRENT: usize = 4; +/// Callback fired after each chunk completes during +/// [`sync_shielded_notes`](super::sync_shielded_notes). +/// +/// Arguments: `(cumulative_notes_scanned, latest_block_height)`. +/// +/// Used to surface live progress during a long initial sync (a cold +/// sync of a 1M-note pool can run >20 min in a single call; without +/// this callback there's no signal between start and end). +/// +/// The callback is fired from inside the sliding-window chunk loop on +/// the same task that drives the sync — it should be cheap. If it +/// needs to do anything expensive (DB write, async work), forward via +/// a channel and process out-of-band. +pub type ProgressCallback = Arc; + /// Configuration for [`sync_shielded_notes`](super::sync_shielded_notes). +#[derive(Clone)] pub struct ShieldedSyncConfig { /// Maximum number of chunk queries in flight at once (default: 4). pub max_concurrent: usize, /// Request settings forwarded to each individual fetch call. pub request_settings: RequestSettings, + /// Optional progress callback — see [`ProgressCallback`]. + /// `None` (default) disables progress reporting. + pub on_chunk_completed: Option, } impl Default for ShieldedSyncConfig { @@ -20,10 +40,50 @@ impl Default for ShieldedSyncConfig { Self { max_concurrent: DEFAULT_MAX_CONCURRENT, request_settings: RequestSettings::default(), + on_chunk_completed: None, } } } +/// One chunk of encrypted notes emitted by the streaming sync entry +/// point [`sync_shielded_notes_stream`](super::sync_shielded_notes::sync_shielded_notes_stream). +/// +/// Batches are yielded **in strictly ascending tree-position order** +/// (`start_index`) even when the underlying network fetches complete +/// out of order — the stream's reorder buffer holds an early-finishing +/// later chunk until every predecessor has been emitted. This lets the +/// consumer append commitments to a position-ordered Merkle tree the +/// instant a contiguous chunk is ready, overlapping tree-append with +/// later network fetches. +pub struct ShieldedChunkBatch { + /// Chunk start position in the commitment tree (tree order). Always + /// a multiple of the fetch size for non-final chunks. + pub start_index: u64, + /// Raw encrypted notes for this chunk, in tree order. The note at + /// vec index `i` has global tree position `start_index + i`. + pub notes: Vec, + /// Driver-IVK trial-decryption hits within this chunk (the notes + /// that decrypted under the `ivk` passed to the stream). Positions + /// are absolute (already offset by `start_index`). + pub decrypted: Vec, + /// Platform block height reported by the response that produced + /// this chunk. + pub block_height: u64, + /// True for the final (buffer) chunk — a short read + /// (`notes.len() < fetch_size`) that signals end-of-stream. The + /// BulkAppendTree buffer chunk is mutable and may receive more + /// notes before the next sync, so the consumer resumes from this + /// chunk's `start_index` next pass. + pub is_partial: bool, + /// On-chain total number of notes in the shielded `CommitmentTree` at + /// the block this chunk's proof was taken against — the sync + /// progress-bar denominator. Extracted from the SAME note-fetch proof + /// (no separate RPC), so it arrives with the very first batch and is + /// stable across the sync; carrying it per-batch is intentional and + /// cheap. + pub total_count: u64, +} + /// A note that was successfully decrypted (belongs to the viewer). pub struct DecryptedNote { /// Global position of this note in the commitment tree. @@ -51,4 +111,10 @@ pub struct ShieldedSyncResult { pub total_notes_scanned: u64, /// Platform block height at the time of the most recent chunk response. pub block_height: u64, + /// On-chain total number of notes in the shielded `CommitmentTree` — the + /// sync progress-bar denominator. Extracted from the note-fetch proofs + /// (no separate RPC); taken as the max/last-seen across the sync's + /// batches. `0` if no chunk was fetched (e.g. an aligned start past the + /// end of the tree returned nothing). + pub total_count: u64, } diff --git a/packages/rs-sdk/src/sdk.rs b/packages/rs-sdk/src/sdk.rs index acc662fb2f0..adf7c7d2e68 100644 --- a/packages/rs-sdk/src/sdk.rs +++ b/packages/rs-sdk/src/sdk.rs @@ -25,7 +25,7 @@ use drive_proof_verifier::FromProof; pub use http::Uri; #[cfg(feature = "mocks")] use rs_dapi_client::mock::MockDapiClient; -use rs_dapi_client::Address; +pub use rs_dapi_client::Address; pub use rs_dapi_client::AddressList; pub use rs_dapi_client::RequestSettings; use rs_dapi_client::{ diff --git a/packages/rs-sdk/tests/fetch/tokens/token_pre_programmed_distributions.rs b/packages/rs-sdk/tests/fetch/tokens/token_pre_programmed_distributions.rs index 5ec7a4fdca9..9ebb653d6c0 100644 --- a/packages/rs-sdk/tests/fetch/tokens/token_pre_programmed_distributions.rs +++ b/packages/rs-sdk/tests/fetch/tokens/token_pre_programmed_distributions.rs @@ -1,5 +1,5 @@ // TODO: Generate test vectors by running against a devnet: -// yarn reset && SDK_TEST_DATA=true yarn start +// yarn reset && yarn start # `yarn setup` already enables SDK_TEST_DATA in local config // ./packages/rs-sdk/scripts/generate_test_vectors.sh test_token_pre_programmed_distributions use crate::fetch::common::setup_logs; diff --git a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManager.swift b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManager.swift index e26f0865133..0540dd48d8a 100644 --- a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManager.swift +++ b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManager.swift @@ -3,6 +3,19 @@ import SwiftData import Combine import DashSDKFFI +/// Lock-guarded monotonic generation counter, safe to read and bump from +/// any thread. Used to drop shielded sync completion events that belong +/// to a generation already superseded by a `stop`/`clear`, even when a +/// restart happens in the same `@MainActor` turn (a plain boolean gate +/// can't, because the restart re-opens the gate before the stale, +/// previously-enqueued completion task runs). +final class ShieldedSyncGenerationCounter: @unchecked Sendable { + private let lock = NSLock() + private var value: UInt64 = 0 + func current() -> UInt64 { lock.withLock { value } } + @discardableResult func bump() -> UInt64 { lock.withLock { value &+= 1; return value } } +} + /// The one thing SwiftUI needs for all wallet operations. /// /// Owns the Rust-side `PlatformWalletManager` handle which drives: @@ -46,21 +59,54 @@ public class PlatformWalletManager: ObservableObject { /// Last completed shielded sync event emitted by Rust. @Published public internal(set) var lastShieldedSyncEvent: ShieldedSyncEvent? - /// When true, `handleShieldedSyncCompleted` drops incoming events - /// instead of publishing them. Set by `stopShieldedSync` / - /// `clearShielded` (after the Rust drain returns) and cleared by any - /// sync-start (`startShieldedSync` / `syncShieldedNow`). The Rust - /// quiesce barrier guarantees no persistence after stop/clear, but - /// the completion callback is re-dispatched onto this `@MainActor`, - /// so a final, already-dispatched event can land just after stop/ - /// clear returns; this gate keeps the published `lastShieldedSyncEvent` - /// honest for every SDK consumer (not just the example app). Both - /// stop/clear are synchronous on the main actor, so the flag is set - /// before the enqueued trailing-event task can run. + /// Cumulative number of encrypted notes scanned in the **current** + /// in-flight shielded sync pass, published once per chunk (~every + /// 2048 notes) via the Rust-side progress callback. Nil between + /// passes. Lets UI render a live counter / `ProgressView` during + /// the cold sync of a large pool (1M notes can take 20+ min in a + /// single SDK call; without this there's no signal between start + /// and end). + /// + /// Paired with `currentShieldedSyncBlockHeight` — emitted in the + /// same callback. They update together; the chain-tip number lets + /// the UI estimate "still N blocks behind". + @Published public internal(set) var currentShieldedSyncScanned: UInt64? + @Published public internal(set) var currentShieldedSyncBlockHeight: UInt64? + + /// Cumulative count of note commitments appended to the local + /// Orchard commitment tree in the **current** in-flight shielded + /// sync pass — the "checked / committed-to-tree" signal, distinct + /// from `currentShieldedSyncScanned` (which counts *downloaded* + /// notes). Published once per committed batch via the Rust-side + /// tree-progress callback. Nil between passes. + /// + /// Paired with `currentShieldedTreeTotal` — emitted in the same + /// callback. `currentShieldedTreeTotal == 0` (or nil) means the + /// total is indeterminate; the UI should render a spinner rather + /// than a determinate bar. + @Published public internal(set) var currentShieldedTreeCommitted: UInt64? + @Published public internal(set) var currentShieldedTreeTotal: UInt64? + + /// Monotonic generation for shielded sync passes. Each `stop`/`clear` + /// bumps it; the FFI completion callback snapshots the generation at + /// enqueue time and `handleShieldedSyncCompleted` drops any event whose + /// snapshot no longer matches the current generation. + /// + /// The Rust quiesce barrier guarantees no persistence after stop/clear, + /// but the completion callback is re-dispatched onto this `@MainActor`, + /// so a final, already-dispatched event can land just after stop/clear + /// returns. A plain boolean gate is bypassable: a caller can stop (set + /// the flag) and restart (clear the flag) in the same actor turn, which + /// re-opens the gate before the stale, previously-enqueued completion + /// task runs — so the old event leaks into the new run. Tying + /// suppression to a generation closes that race: the stale task carries + /// the pre-stop generation, the restart does not reset the counter, so + /// the snapshot mismatches and the event is dropped even on a same-turn + /// restart. /// - /// `internal` (not `private`) because the shielded lifecycle methods - /// that read/write it live in an extension in a separate file. - var suppressShieldedCompletionEvents: Bool = false + /// `nonisolated` + lock-guarded so the FFI callback thread can snapshot + /// it without hopping onto the main actor first. + nonisolated let shieldedSyncGeneration = ShieldedSyncGenerationCounter() /// All wallets currently held by the Rust-side /// `PlatformWalletManager`, keyed by the 32-byte wallet id. diff --git a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerAddressSync.swift b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerAddressSync.swift index c0023d00372..d26d47aefd6 100644 --- a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerAddressSync.swift +++ b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerAddressSync.swift @@ -49,6 +49,8 @@ final class PlatformWalletEventHandler { callbacks.context = Unmanaged.passUnretained(self).toOpaque() callbacks.on_platform_address_sync_completed_fn = platformAddressSyncCompletedCallback callbacks.on_shielded_sync_completed_fn = shieldedSyncCompletedCallback + callbacks.on_shielded_sync_progress_fn = shieldedSyncProgressCallback + callbacks.on_shielded_tree_progress_fn = shieldedTreeProgressCallback return callbacks } } diff --git a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerShieldedSync.swift b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerShieldedSync.swift index 52d56bc60ad..11186008cf8 100644 --- a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerShieldedSync.swift +++ b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerShieldedSync.swift @@ -58,13 +58,76 @@ public struct ShieldedSyncEvent: Sendable { } extension PlatformWalletManager { - func handleShieldedSyncCompleted(_ event: ShieldedSyncEvent) { - // Drop a trailing event that the Rust drain already dispatched - // but the main actor only delivers after stop/clear returned - // (see `suppressShieldedCompletionEvents`). Any sync-start clears - // the flag, so legitimate events are never suppressed. - guard !suppressShieldedCompletionEvents else { return } + func handleShieldedSyncCompleted(_ event: ShieldedSyncEvent, generation: UInt64) { + // Drop a trailing event that the Rust drain already dispatched but + // the main actor only delivers after stop/clear returned. The FFI + // callback snapshots `shieldedSyncGeneration` at enqueue time; a + // stop/clear bumps the counter, so a stale event's snapshot no + // longer matches and is dropped — even if a restart happened in the + // same actor turn (the restart does not reset the counter). + guard generation == shieldedSyncGeneration.current() else { return } lastShieldedSyncEvent = event + // A completed pass means the per-chunk progress counter for + // this pass is no longer meaningful — clear so the next pass + // starts from nil. Also matches the false→true edge UI gating + // in ShieldedService's currentSyncElapsed timer. + resetCurrentShieldedProgress() + } + + /// Clear the four per-pass live-progress mirrors so the next pass + /// starts from nil. Routed through by every path that ends a pass: + /// the normal completion (`handleShieldedSyncCompleted`) plus the + /// `stopShieldedSync()` / `clearShielded()` paths that suppress the + /// trailing completion event — without this, a pass stopped/cleared + /// mid-flight would leave the last published `currentShielded*` + /// values visible (stale UI) between passes. Main-actor-isolated + /// like every other member of this `@MainActor` class. + private func resetCurrentShieldedProgress() { + currentShieldedSyncScanned = nil + currentShieldedSyncBlockHeight = nil + currentShieldedTreeCommitted = nil + currentShieldedTreeTotal = nil + } + + /// Per-chunk progress callback. Fires once per ~2048 notes + /// processed during a cold sync; bridged here from the C + /// trampoline `shieldedSyncProgressCallback`. Cheap publish; UI + /// gets it through ShieldedService. + /// + /// Generation-guarded like `handleShieldedSyncCompleted`: a stale + /// progress hop delivered after a stop/clear bumped the generation + /// must be dropped so it can't re-publish phantom progress over the + /// `resetCurrentShieldedProgress()` mirrors the stop/clear just reset. + func handleShieldedSyncProgress( + cumulativeScanned: UInt64, + blockHeight: UInt64, + generation: UInt64 + ) { + guard generation == shieldedSyncGeneration.current() else { return } + currentShieldedSyncScanned = cumulativeScanned + currentShieldedSyncBlockHeight = blockHeight + } + + /// Per-batch tree-progress callback — the "checked / + /// committed-to-tree" signal. Fires once per committed batch as + /// commitments are appended to the local Orchard tree; bridged + /// here from the C trampoline `shieldedTreeProgressCallback`. + /// `total == 0` means the on-chain total is indeterminate. Cheap + /// publish; UI gets it through ShieldedService. + /// + /// Generation-guarded like `handleShieldedSyncCompleted`: a stale + /// tree-progress hop delivered after a stop/clear bumped the + /// generation must be dropped so it can't re-publish phantom progress + /// over the `resetCurrentShieldedProgress()` mirrors the stop/clear + /// just reset. + func handleShieldedTreeProgress( + committed: UInt64, + total: UInt64, + generation: UInt64 + ) { + guard generation == shieldedSyncGeneration.current() else { return } + currentShieldedTreeCommitted = committed + currentShieldedTreeTotal = total } /// Derive Orchard keys for `walletId` from the host-side mnemonic @@ -181,8 +244,10 @@ extension PlatformWalletManager { if let intervalSeconds { try setShieldedSyncInterval(seconds: intervalSeconds) } - // A new sync run should publish its completion events again. - suppressShieldedCompletionEvents = false + // No generation reset needed: events emitted by this new run + // snapshot the current generation, so they pass the guard. A + // trailing event from a prior, stopped run still carries the older + // generation and is dropped. try platform_wallet_manager_shielded_sync_start(handle).check() } @@ -193,9 +258,14 @@ extension PlatformWalletManager { ) } try platform_wallet_manager_shielded_sync_stop(handle).check() - // The Rust drain returned; suppress any trailing completion - // event the main actor delivers after this point. - suppressShieldedCompletionEvents = true + // The Rust drain returned; bump the generation so any trailing + // completion event the main actor delivers after this point is + // dropped (its snapshot predates this bump). + shieldedSyncGeneration.bump() + // The dropped completion would normally clear the per-pass + // progress mirrors; do it here so a pass stopped mid-flight + // doesn't leave stale `currentShielded*` values on the UI. + resetCurrentShieldedProgress() } /// Reset the Rust-side shielded state on this manager: @@ -219,10 +289,15 @@ extension PlatformWalletManager { ) } try platform_wallet_manager_shielded_clear(handle).check() - // The Rust drain returned; suppress any trailing completion - // event the main actor delivers after Clear (it would otherwise - // briefly repopulate the mirror the host is about to wipe). - suppressShieldedCompletionEvents = true + // The Rust drain returned; bump the generation so any trailing + // completion event the main actor delivers after Clear is dropped + // (it would otherwise briefly repopulate the mirror the host is + // about to wipe). + shieldedSyncGeneration.bump() + // The dropped completion would normally clear the per-pass + // progress mirrors; do it here so a pass cleared mid-flight + // doesn't leave stale `currentShielded*` values on the UI. + resetCurrentShieldedProgress() } public func isShieldedSyncRunning() throws -> Bool { @@ -273,9 +348,9 @@ extension PlatformWalletManager { "PlatformWalletManager not configured" ) } - // A user-initiated sync should publish its completion event even - // if a prior stop/clear had suppressed events. - suppressShieldedCompletionEvents = false + // No generation reset needed: this run's completion event snapshots + // the current generation and passes the guard, while a trailing + // event from a prior stopped run still carries the older generation. let handle = self.handle try await Task.detached(priority: .userInitiated) { try platform_wallet_manager_shielded_sync_sync_now(handle).check() @@ -586,7 +661,70 @@ func shieldedSyncCompletedCallback( walletResults: results ) + // Snapshot the generation now, on the FFI callback thread, BEFORE the + // event is enqueued onto the main actor. A subsequent stop/clear bumps + // the counter, so this trailing event is dropped when it finally runs. + let generation = handler.manager?.shieldedSyncGeneration.current() ?? 0 + + Task { @MainActor [weak manager = handler.manager] in + manager?.handleShieldedSyncCompleted(event, generation: generation) + } +} + +/// C trampoline matching `EventHandlerCallbacks.on_shielded_sync_progress_fn`. +/// Fires once per ~2048 notes processed during a cold sync. Cheap — +/// just hops to the main actor and publishes the snapshot. +func shieldedSyncProgressCallback( + context: UnsafeMutableRawPointer?, + cumulativeScanned: UInt64, + blockHeight: UInt64 +) { + guard let context else { return } + + let handler = Unmanaged + .fromOpaque(context) + .takeUnretainedValue() + + // Snapshot the generation now, on the FFI callback thread, BEFORE the + // event is enqueued onto the main actor. A subsequent stop/clear bumps + // the counter, so this trailing event is dropped when it finally runs. + let generation = handler.manager?.shieldedSyncGeneration.current() ?? 0 + + Task { @MainActor [weak manager = handler.manager] in + manager?.handleShieldedSyncProgress( + cumulativeScanned: cumulativeScanned, + blockHeight: blockHeight, + generation: generation + ) + } +} + +/// C trampoline matching `EventHandlerCallbacks.on_shielded_tree_progress_fn`. +/// The "checked / committed-to-tree" signal — fires once per committed +/// batch as commitments are appended to the local Orchard tree. +/// `totalTarget == 0` means the on-chain total is indeterminate. Cheap — +/// just hops to the main actor and publishes the snapshot. +func shieldedTreeProgressCallback( + context: UnsafeMutableRawPointer?, + leavesCommitted: UInt64, + totalTarget: UInt64 +) { + guard let context else { return } + + let handler = Unmanaged + .fromOpaque(context) + .takeUnretainedValue() + + // Snapshot the generation now, on the FFI callback thread, BEFORE the + // event is enqueued onto the main actor. A subsequent stop/clear bumps + // the counter, so this trailing event is dropped when it finally runs. + let generation = handler.manager?.shieldedSyncGeneration.current() ?? 0 + Task { @MainActor [weak manager = handler.manager] in - manager?.handleShieldedSyncCompleted(event) + manager?.handleShieldedTreeProgress( + committed: leavesCommitted, + total: totalTarget, + generation: generation + ) } } diff --git a/packages/swift-sdk/SwiftExampleApp/SwiftExampleApp.xcodeproj/project.pbxproj b/packages/swift-sdk/SwiftExampleApp/SwiftExampleApp.xcodeproj/project.pbxproj index f9eb115d3e5..93529314cc1 100644 --- a/packages/swift-sdk/SwiftExampleApp/SwiftExampleApp.xcodeproj/project.pbxproj +++ b/packages/swift-sdk/SwiftExampleApp/SwiftExampleApp.xcodeproj/project.pbxproj @@ -461,7 +461,8 @@ CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_TEAM = 44RJ69WHFF; ENABLE_PREVIEWS = YES; - GENERATE_INFOPLIST_FILE = YES; + GENERATE_INFOPLIST_FILE = NO; + INFOPLIST_FILE = Info.plist; INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES; INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; INFOPLIST_KEY_UILaunchScreen_Generation = YES; diff --git a/packages/swift-sdk/SwiftExampleApp/SwiftExampleApp/Core/Services/ShieldedService.swift b/packages/swift-sdk/SwiftExampleApp/SwiftExampleApp/Core/Services/ShieldedService.swift index 8100283058c..a2037e94131 100644 --- a/packages/swift-sdk/SwiftExampleApp/SwiftExampleApp/Core/Services/ShieldedService.swift +++ b/packages/swift-sdk/SwiftExampleApp/SwiftExampleApp/Core/Services/ShieldedService.swift @@ -101,6 +101,75 @@ class ShieldedService: ObservableObject { /// Subscription to `walletManager.$lastShieldedSyncEvent`. private var syncEventCancellable: AnyCancellable? + // MARK: - Timing instrumentation + // + // Surfaces wall-clock of each sync pass so devnet stress tests + // (1M shielded notes via `dashpay/drive:3.1-shielded.*`) can be + // measured from the iOS client. See + // `docs/shielded-sync-timing-spec.md` for the design. + + /// Wall-clock of the most recent NON-cooldown completed sync + /// pass. Nil until the first such pass after `bind()`. + @Published var lastSyncDuration: TimeInterval? + + /// Maximum wall-clock observed across every completed sync pass + /// since the last `bind()` / `reset()` / `clearLocalState`. Used + /// to preserve the cold-sync headline number when subsequent + /// steady-state passes (3 s deltas) would otherwise clobber + /// `lastSyncDuration` in the UI. Nil until the first completed + /// pass. + @Published var longestSyncDuration: TimeInterval? + + /// Running wall-clock of the in-flight sync pass. Updated by a + /// 1Hz timer while `isSyncing == true`; nil otherwise. + @Published var currentSyncElapsed: TimeInterval? + + /// Cumulative encrypted notes scanned in the in-flight pass. + /// Republished from `PlatformWalletManager.currentShieldedSyncScanned` + /// — fired once per chunk (~2048 notes) by the Rust progress + /// callback. Nil between passes. Lets the UI render a live + /// counter / ProgressView during a cold sync. + @Published var currentSyncScanned: UInt64? + + /// Latest Platform block height observed during the in-flight + /// pass. Pairs with `currentSyncScanned` (same callback). + @Published var currentSyncBlockHeight: UInt64? + + /// Cumulative note commitments appended to the local Orchard tree + /// in the in-flight pass — the "checked / committed-to-tree" + /// signal, distinct from `currentSyncScanned` (which counts + /// *downloaded* notes). Republished from + /// `PlatformWalletManager.currentShieldedTreeCommitted`, fired once + /// per committed batch by the Rust tree-progress callback. Nil + /// between passes. + @Published var currentTreeCommitted: UInt64? + + /// On-chain MMR total leaf count, the denominator for both the + /// "downloaded" and "checked" bars (total notes == total leaves). + /// Pairs with `currentTreeCommitted` (same callback). A value of 0 + /// (or nil) means the total is indeterminate — render a spinner + /// rather than a determinate bar. + @Published var currentTreeTotal: UInt64? + + /// Subscription to `walletManager.$currentShieldedSyncScanned` + /// and `…BlockHeight` for live progress. Created in `bind` / + /// `bind`, dropped in `reset` / `clearLocalState`. + private var progressCancellable: AnyCancellable? + + /// Subscription to `walletManager.$currentShieldedTreeCommitted` + /// and `…Total` for the "checked / committed-to-tree" bar. Created + /// in `bind`, dropped in `reset` / `clearLocalState`. + private var treeProgressCancellable: AnyCancellable? + + /// `Date()` at the moment `isSyncing` flipped false → true. + /// Drives both `lastSyncDuration` (at completion) and + /// `currentSyncElapsed` (live). + private var currentSyncStartedAt: Date? + + /// 1Hz timer that ticks `currentSyncElapsed` while syncing. + /// Started on false→true edge, invalidated on true→false edge. + private var syncTickTimer: Timer? + // MARK: - Lifecycle /// Bind the service to a wallet. Drives `bindShielded` on the @@ -131,6 +200,8 @@ class ShieldedService: ObservableObject { self.resolver = resolver self.syncStateCancellable?.cancel() self.syncEventCancellable?.cancel() + self.progressCancellable?.cancel() + self.treeProgressCancellable?.cancel() // Clear the previous wallet's snapshot up front. Without // this, switching wallets (or a failed rebind) leaves the @@ -154,6 +225,16 @@ class ShieldedService: ObservableObject { totalScanned = 0 totalNewNotes = 0 totalNewlySpent = 0 + lastSyncDuration = nil + longestSyncDuration = nil + currentSyncElapsed = nil + currentSyncStartedAt = nil + currentSyncScanned = nil + currentSyncBlockHeight = nil + currentTreeCommitted = nil + currentTreeTotal = nil + syncTickTimer?.invalidate() + syncTickTimer = nil let dbPath = Self.dbPath(for: network) let sortedAccounts = Array(Set(accounts)).sorted() @@ -207,8 +288,38 @@ class ShieldedService: ObservableObject { } syncStateCancellable = walletManager.$shieldedSyncIsSyncing - .sink { [weak self] isSyncing in - self?.isSyncing = isSyncing + .sink { [weak self] newValue in + guard let self else { return } + let wasSyncing = self.isSyncing + self.isSyncing = newValue + // Detect false → true edge. `.sink` fires on every + // republished value, including duplicates, so we + // gate on the previous mirror to avoid log spam. The + // timing bracket itself is opened by + // `beginSyncTimingIfNeeded()`, which is idempotent — + // `manualSync()`'s fast path may have already opened it + // for a pass that completes before this publisher flips. + if newValue && !wasSyncing { + self.beginSyncTimingIfNeeded() + SDKLogger.log( + "Shielded sync started", + minimumLevel: .medium + ) + } + // Detect true → false edge. Tear down the ticker and + // zero the live elapsed, but do NOT clear + // `currentSyncStartedAt` here — + // `handleShieldedSyncEvent` still needs it to compute + // `lastSyncDuration` and calls `endSyncTiming()` itself + // on every terminal path. This branch only covers the + // edge where the publisher flips false with no paired + // event (defensive); the event handler is the + // authoritative close. + if !newValue && wasSyncing { + self.syncTickTimer?.invalidate() + self.syncTickTimer = nil + self.currentSyncElapsed = nil + } } syncEventCancellable = walletManager.$lastShieldedSyncEvent @@ -216,6 +327,30 @@ class ShieldedService: ObservableObject { guard let self, let event else { return } self.handleShieldedSyncEvent(event) } + + // Bridge per-chunk progress from the manager. Pair + // `currentShieldedSyncScanned` and `…BlockHeight`; they're + // emitted by the same Rust callback so a `combineLatest` + // round-trips them coherently into our two @Published mirrors. + progressCancellable = walletManager.$currentShieldedSyncScanned + .combineLatest(walletManager.$currentShieldedSyncBlockHeight) + .sink { [weak self] scanned, height in + guard let self else { return } + self.currentSyncScanned = scanned + self.currentSyncBlockHeight = height + } + + // Bridge the second "checked / committed-to-tree" signal from + // the manager. Pair `currentShieldedTreeCommitted` and `…Total`; + // they're emitted by the same Rust callback so a `combineLatest` + // round-trips them coherently into our two @Published mirrors. + treeProgressCancellable = walletManager.$currentShieldedTreeCommitted + .combineLatest(walletManager.$currentShieldedTreeTotal) + .sink { [weak self] committed, total in + guard let self else { return } + self.currentTreeCommitted = committed + self.currentTreeTotal = total + } } /// Re-bind the singleton service to a different wallet using the @@ -295,6 +430,14 @@ class ShieldedService: ObservableObject { isSyncing = true lastError = nil + // Open the timing bracket here too, not just on the + // `$shieldedSyncIsSyncing` false→true edge. A fast pass + // (e.g. empty-tree sync) can complete before that publisher + // ever flips, so without this `currentSyncStartedAt` would be + // nil at completion and `lastSyncDuration` would be dropped. + // `beginSyncTimingIfNeeded()` is idempotent, so if the + // publisher does flip first it simply no-ops there. + beginSyncTimingIfNeeded() defer { isSyncing = false } do { try await walletManager.syncShieldedNow() @@ -346,6 +489,8 @@ class ShieldedService: ObservableObject { func reset() { syncStateCancellable?.cancel() syncEventCancellable?.cancel() + progressCancellable?.cancel() + treeProgressCancellable?.cancel() walletManager = nil boundWalletId = nil isSyncing = false @@ -362,6 +507,16 @@ class ShieldedService: ObservableObject { totalScanned = 0 totalNewNotes = 0 totalNewlySpent = 0 + lastSyncDuration = nil + longestSyncDuration = nil + currentSyncElapsed = nil + currentSyncStartedAt = nil + currentSyncScanned = nil + currentSyncBlockHeight = nil + currentTreeCommitted = nil + currentTreeTotal = nil + syncTickTimer?.invalidate() + syncTickTimer = nil } /// Wipe every wallet's persisted shielded state and stop. The @@ -479,6 +634,8 @@ class ShieldedService: ObservableObject { // them and leave the user stranded on this screen. syncStateCancellable?.cancel() syncEventCancellable?.cancel() + progressCancellable?.cancel() + treeProgressCancellable?.cancel() isBound = false isSyncing = false shieldedBalance = 0 @@ -492,6 +649,61 @@ class ShieldedService: ObservableObject { totalScanned = 0 totalNewNotes = 0 totalNewlySpent = 0 + lastSyncDuration = nil + longestSyncDuration = nil + currentSyncElapsed = nil + currentSyncStartedAt = nil + currentSyncScanned = nil + currentSyncBlockHeight = nil + currentTreeCommitted = nil + currentTreeTotal = nil + syncTickTimer?.invalidate() + syncTickTimer = nil + } + + // MARK: - Sync timing brackets + + /// Open the timing bracket for a sync pass: stamp the start, zero + /// the live elapsed, and start the 1 Hz ticker. Idempotent via the + /// `currentSyncStartedAt == nil` guard so it can be called from + /// either the `$shieldedSyncIsSyncing` false→true edge OR the + /// `manualSync()` fast path — whichever observes the pass starting + /// first wins, and the other no-ops. This closes the gap where a + /// fast pass completes before the publisher flips, dropping + /// `lastSyncDuration`. + private func beginSyncTimingIfNeeded() { + guard currentSyncStartedAt == nil else { return } + currentSyncStartedAt = Date() + currentSyncElapsed = 0 + syncTickTimer?.invalidate() + syncTickTimer = Timer.scheduledTimer( + withTimeInterval: 1.0, + repeats: true + ) { [weak self] _ in + Task { @MainActor [weak self] in + guard let self, + let started = self.currentSyncStartedAt + else { return } + self.currentSyncElapsed = max( + 0, + Date().timeIntervalSince(started) + ) + } + } + } + + /// Close the timing bracket: tear down the ticker and clear both + /// the live elapsed and the start stamp. Every terminal flow must + /// route through this (normal completion, cooldown-skip, skipped, + /// failure) so no stale `currentSyncStartedAt` survives to be + /// reused — and re-misreport — on the next pass's completion. + /// Callers that report `lastSyncDuration` must read + /// `currentSyncStartedAt` BEFORE calling this. + private func endSyncTiming() { + syncTickTimer?.invalidate() + syncTickTimer = nil + currentSyncElapsed = nil + currentSyncStartedAt = nil } // MARK: - Sync event handling @@ -542,14 +754,80 @@ class ShieldedService: ObservableObject { totalScanned += result.totalScanned totalNewNotes += UInt64(result.newNotes) totalNewlySpent += UInt64(result.newlySpent) + + // Record per-pass wall-clock and log it. `Date()` + // here is the Swift-side timestamp of when the + // event handler runs (≈ when isSyncing flipped + // true → false), pairing with `currentSyncStartedAt` + // captured on the false → true edge. Clamp to >= 0 + // defensively — should never be negative with + // Swift-edge endpoints, but if the start timestamp + // is missing (e.g. event arrived without a paired + // start, post-Clear race) we surface nil rather + // than a misleading number. + if let started = currentSyncStartedAt { + let elapsed = max(0, Date().timeIntervalSince(started)) + lastSyncDuration = elapsed + // Preserve the longest pass observed since the + // last reset. Cold sync (~20 min for 1M notes on + // paloma) would otherwise get clobbered by the + // next steady-state pass (~3 s). The cold number + // is the headline measurement; keep it visible. + if let prev = longestSyncDuration { + if elapsed > prev { longestSyncDuration = elapsed } + } else { + longestSyncDuration = elapsed + } + let rateString: String + if elapsed > 0.05 && result.totalScanned > 0 { + let rate = Double(result.totalScanned) / elapsed + rateString = String(format: " rate=%.0f/s", rate) + } else { + rateString = "" + } + SDKLogger.log( + String( + format: "Shielded sync done pass=%d elapsed=%.2fs%@ scanned=%llu new=%u spent=%u balance=%llu", + syncCountSinceLaunch, + elapsed, + rateString, + result.totalScanned, + result.newNotes, + result.newlySpent, + result.balance + ), + minimumLevel: .medium + ) + } else { + lastSyncDuration = nil + SDKLogger.log( + "Shielded sync done (no paired start) pass=\(syncCountSinceLaunch) scanned=\(result.totalScanned) balance=\(result.balance)", + minimumLevel: .medium + ) + } + // Close the timing bracket AFTER reading + // `currentSyncStartedAt` above. Tears down the ticker, + // clears `currentSyncElapsed`, and nils the start stamp. + endSyncTiming() + } else { + // Cooldown-skip terminal: no work ran, so we leave the + // cached balance / counters alone — but the timing + // bracket still has to close, otherwise a stale start + // stamp would be reused on the next pass's completion. + endSyncTiming() } } else if result.skipped { // Skipped means the wallet hasn't been bound yet on the // Rust side. The UI can prompt the user to retry the - // bind step. + // bind step. Close the timing bracket so no stale start + // stamp survives to the next pass. isBound = false + endSyncTiming() } else { + // Failure terminal: surface the error and close the timing + // bracket so the stale start stamp isn't reused next pass. lastError = result.errorMessage ?? "Shielded sync failed" + endSyncTiming() } } diff --git a/packages/swift-sdk/SwiftExampleApp/SwiftExampleApp/Core/Views/CoreContentView.swift b/packages/swift-sdk/SwiftExampleApp/SwiftExampleApp/Core/Views/CoreContentView.swift index ec6036715fb..74ff9cb85d0 100644 --- a/packages/swift-sdk/SwiftExampleApp/SwiftExampleApp/Core/Views/CoreContentView.swift +++ b/packages/swift-sdk/SwiftExampleApp/SwiftExampleApp/Core/Views/CoreContentView.swift @@ -432,6 +432,84 @@ var body: some View { // bound wallet. ShieldedNetworkSummaryRows(walletIds: walletIdsOnNetwork) + // Per-pass wall-clock timing. While a sync is + // in-flight, shows the live ticker (driven by a + // 1Hz timer on ShieldedService). After + // completion, shows the most recent non-cooldown + // pass duration. Mono digits keep the number + // readable as it ticks during long initial + // syncs (e.g. 10 min at N=1M). See + // `docs/shielded-sync-timing-spec.md`. + if shieldedService.isSyncing, + let elapsed = shieldedService.currentSyncElapsed { + VStack(alignment: .leading, spacing: 4) { + HStack { + Text("Syncing… elapsed") + .font(.subheadline) + .foregroundColor(.secondary) + Spacer() + Text(String(format: "%.1f s", elapsed)) + .font(.caption) + .fontWeight(.medium) + .monospacedDigit() + } + // Dual per-pass progress (P1.2). Two bars + // share the same denominator — the on-chain + // MMR total leaf count (total notes == total + // leaves) — carried straight from Rust in the + // tree-progress callback's second arg. The + // "Downloaded" bar tracks notes pulled off the + // wire; the "Checked" bar tracks commitments + // appended to the local Orchard tree. When the + // total is unknown (RPC unavailable, or before + // the first tree batch lands) `currentTreeTotal` + // is nil/0 and each bar falls back to an + // indeterminate spinner with the raw count. + if shieldedService.currentSyncScanned != nil + || shieldedService.currentTreeCommitted != nil { + ShieldedDualProgressRows( + downloaded: shieldedService.currentSyncScanned, + checked: shieldedService.currentTreeCommitted, + total: shieldedService.currentTreeTotal + ) + } + } + } else if let duration = shieldedService.lastSyncDuration { + HStack { + Text("Last sync duration") + .font(.subheadline) + .foregroundColor(.secondary) + Spacer() + Text(String(format: "%.2f s", max(0, duration))) + .font(.caption) + .fontWeight(.medium) + .monospacedDigit() + } + } + + // "Longest pass" row — survives short steady-state + // re-passes so the cold-sync wall clock (the + // headline number for 1M-note devnet stress) stays + // visible after subsequent fast deltas overwrite + // `lastSyncDuration`. Only rendered when it + // actually exceeds the most recent pass to avoid + // redundant display. + if let longest = shieldedService.longestSyncDuration, + let last = shieldedService.lastSyncDuration, + longest > last + 0.05 { + HStack { + Text("Longest pass") + .font(.subheadline) + .foregroundColor(.secondary) + Spacer() + Text(String(format: "%.2f s", longest)) + .font(.caption) + .fontWeight(.medium) + .monospacedDigit() + .foregroundColor(.secondary) + } + } + // Sync counters since launch — `total_scanned` // is the wire-level encrypted-note count (every // pass), while new + spent are the wallet-side @@ -1348,3 +1426,80 @@ private struct ShieldedNetworkSummaryRows: View { } } } + +/// Dual live progress for an in-flight shielded sync pass: a +/// "Downloaded" bar (notes pulled off the wire) stacked over a +/// "Checked" bar (commitments appended to the local Orchard tree). +/// +/// Both bars share the same denominator — `total`, the on-chain MMR +/// total leaf count carried straight from Rust in the tree-progress +/// callback (total notes == total leaves). No Swift-side math derives +/// it. When `total` is nil or 0 the total is indeterminate (the count +/// RPC was unavailable, or no tree batch has landed yet) and each bar +/// degrades to an indeterminate spinner alongside its raw count. +private struct ShieldedDualProgressRows: View { + /// Cumulative notes downloaded this pass; nil before the first + /// download chunk. + let downloaded: UInt64? + /// Cumulative commitments appended to the tree this pass; nil + /// before the first committed batch. + let checked: UInt64? + /// Shared denominator (on-chain MMR total). nil/0 ⇒ indeterminate. + let total: UInt64? + + /// Determinate only when Rust handed us a positive total. + private var hasTotal: Bool { + if let total, total > 0 { return true } + return false + } + + var body: some View { + VStack(alignment: .leading, spacing: 6) { + progressRow(label: "Downloaded", value: downloaded) + progressRow(label: "Checked", value: checked) + } + } + + @ViewBuilder + private func progressRow(label: String, value: UInt64?) -> some View { + VStack(alignment: .leading, spacing: 2) { + HStack { + Text(label) + .font(.caption2) + .foregroundColor(.secondary) + Spacer() + Text(countText(value: value)) + .font(.caption2) + .monospacedDigit() + .foregroundColor(.secondary) + } + if hasTotal, let value, let total { + // Clamp so a value that briefly overshoots the cached + // total (batch lands before the denominator refreshes) + // can't push the bar past 1.0. + ProgressView( + value: Double(min(value, total)), + total: Double(total) + ) + .progressViewStyle(.linear) + .tint(.purple) + } else { + // Indeterminate: total unknown ⇒ spinner, not a fake bar. + ProgressView() + .progressViewStyle(.linear) + .tint(.purple) + } + } + } + + /// "12,288 / 1,000,000 notes" when the total is known, else + /// "12,288 notes" — matches the existing scanned-count presentation. + private func countText(value: UInt64?) -> String { + let count = value ?? 0 + let countStr = count.formatted(.number) + if hasTotal, let total { + return "\(countStr) / \(total.formatted(.number)) notes" + } + return "\(countStr) notes" + } +} diff --git a/packages/swift-sdk/SwiftExampleApp/SwiftExampleAppTests/ShieldedSyncGenerationTests.swift b/packages/swift-sdk/SwiftExampleApp/SwiftExampleAppTests/ShieldedSyncGenerationTests.swift new file mode 100644 index 00000000000..0d9f10a17a9 --- /dev/null +++ b/packages/swift-sdk/SwiftExampleApp/SwiftExampleAppTests/ShieldedSyncGenerationTests.swift @@ -0,0 +1,234 @@ +import XCTest +@testable import SwiftDashSDK + +/// Regression coverage for the shielded-sync generation guard in +/// `PlatformWalletManager.handleShieldedSyncCompleted(_:generation:)`. +/// +/// The guard exists because a shielded-sync completion event is snapshotted +/// with the current generation on the FFI callback thread, then re-dispatched +/// onto the main actor. A `stop`/`clear` bumps the generation counter, so a +/// trailing event that the main actor delivers *after* stop/clear must be +/// dropped — its snapshot predates the bump. Crucially, a same-turn restart +/// must NOT re-open the gate, because the counter is monotonic and never reset. +/// +/// `handleShieldedSyncCompleted` does not touch the FFI handle — it only reads +/// `shieldedSyncGeneration` and assigns `lastShieldedSyncEvent` — so a bare, +/// unconfigured `PlatformWalletManager()` is safe to drive directly. +/// +/// `PlatformWalletManager` and `handleShieldedSyncCompleted` are main-actor +/// isolated, so the whole suite runs on the main actor. +@MainActor +final class ShieldedSyncGenerationTests: XCTestCase { + + /// Build a deterministic event distinguishable by its timestamp. + private func makeEvent(syncUnixSeconds: UInt64) -> ShieldedSyncEvent { + ShieldedSyncEvent(syncUnixSeconds: syncUnixSeconds, walletResults: []) + } + + /// A completion captured under generation N is dropped once the counter is + /// bumped to N+1 before delivery: `lastShieldedSyncEvent` stays nil. + func testStaleCompletionIsDroppedAfterGenerationBump() { + let manager = PlatformWalletManager() + XCTAssertNil(manager.lastShieldedSyncEvent, "fresh manager has no event") + + // Snapshot the generation at "enqueue" time, the way the FFI callback + // thread does, then advance it (as stop/clear would) before delivery. + let capturedGeneration = manager.shieldedSyncGeneration.current() + manager.shieldedSyncGeneration.bump() + + let staleEvent = makeEvent(syncUnixSeconds: 1_000) + manager.handleShieldedSyncCompleted(staleEvent, generation: capturedGeneration) + + XCTAssertNil( + manager.lastShieldedSyncEvent, + "an event captured under a superseded generation must be dropped" + ) + } + + /// A completion captured under the CURRENT generation is published: + /// `lastShieldedSyncEvent` is set to that exact event. + func testCurrentGenerationCompletionIsPublished() { + let manager = PlatformWalletManager() + + let currentGeneration = manager.shieldedSyncGeneration.current() + let event = makeEvent(syncUnixSeconds: 2_000) + manager.handleShieldedSyncCompleted(event, generation: currentGeneration) + + XCTAssertEqual( + manager.lastShieldedSyncEvent?.syncUnixSeconds, + event.syncUnixSeconds, + "an event captured under the current generation must be published" + ) + } + + /// The exact race the guard closes: a stale event captured under the old + /// generation is dropped even when a same-turn restart follows, and a + /// later event captured under the (unchanged) current generation is still + /// published. Proves `bump()` — not any gate reset — is what invalidates. + func testBumpInvalidatesStaleButNotSubsequentCurrentCompletion() { + let manager = PlatformWalletManager() + + // 1. Capture under the pre-stop generation, then bump (stop/clear). + let staleGeneration = manager.shieldedSyncGeneration.current() + manager.shieldedSyncGeneration.bump() + + // 2. A restart does NOT reset the counter — capture the post-bump + // generation that a fresh run's events would carry. + let liveGeneration = manager.shieldedSyncGeneration.current() + XCTAssertNotEqual( + staleGeneration, + liveGeneration, + "bump must advance the generation so the stale snapshot no longer matches" + ) + + // 3. The trailing stale event (old snapshot) is delivered and dropped. + let staleEvent = makeEvent(syncUnixSeconds: 3_000) + manager.handleShieldedSyncCompleted(staleEvent, generation: staleGeneration) + XCTAssertNil( + manager.lastShieldedSyncEvent, + "the stale completion must not leak into the restarted run" + ) + + // 4. The new run's event (current snapshot) is delivered and published. + let liveEvent = makeEvent(syncUnixSeconds: 4_000) + manager.handleShieldedSyncCompleted(liveEvent, generation: liveGeneration) + XCTAssertEqual( + manager.lastShieldedSyncEvent?.syncUnixSeconds, + liveEvent.syncUnixSeconds, + "the restarted run's completion must be published" + ) + } + + /// A current-generation event followed by a stale (pre-bump) event keeps + /// the first event in place: a late straggler must not clobber valid state. + func testStaleCompletionDoesNotOverwriteAlreadyPublishedEvent() { + let manager = PlatformWalletManager() + + // Publish a valid event under the current generation. + let staleGeneration = manager.shieldedSyncGeneration.current() + let publishedEvent = makeEvent(syncUnixSeconds: 5_000) + manager.handleShieldedSyncCompleted(publishedEvent, generation: staleGeneration) + XCTAssertEqual(manager.lastShieldedSyncEvent?.syncUnixSeconds, publishedEvent.syncUnixSeconds) + + // Bump (stop/clear), then deliver a straggler carrying the old snapshot. + manager.shieldedSyncGeneration.bump() + let straggler = makeEvent(syncUnixSeconds: 6_000) + manager.handleShieldedSyncCompleted(straggler, generation: staleGeneration) + + XCTAssertEqual( + manager.lastShieldedSyncEvent?.syncUnixSeconds, + publishedEvent.syncUnixSeconds, + "a superseded straggler must not overwrite the last valid event" + ) + } + + // MARK: - Per-chunk sync-progress guard + + /// A sync-progress hop captured under generation N is dropped once the + /// counter is bumped to N+1 before delivery: neither + /// `currentShieldedSyncScanned` nor `currentShieldedSyncBlockHeight` is set. + func testStaleSyncProgressIsDroppedAfterGenerationBump() { + let manager = PlatformWalletManager() + XCTAssertNil(manager.currentShieldedSyncScanned, "fresh manager has no scanned mirror") + XCTAssertNil(manager.currentShieldedSyncBlockHeight, "fresh manager has no block-height mirror") + + // Snapshot the generation at "enqueue" time, the way the FFI callback + // thread does, then advance it (as stop/clear would) before delivery. + let capturedGeneration = manager.shieldedSyncGeneration.current() + manager.shieldedSyncGeneration.bump() + + manager.handleShieldedSyncProgress( + cumulativeScanned: 2_048, + blockHeight: 100, + generation: capturedGeneration + ) + + XCTAssertNil( + manager.currentShieldedSyncScanned, + "a progress hop under a superseded generation must not publish scanned" + ) + XCTAssertNil( + manager.currentShieldedSyncBlockHeight, + "a progress hop under a superseded generation must not publish block height" + ) + } + + /// A sync-progress hop captured under the CURRENT generation is published: + /// both `currentShieldedSync*` mirrors are set to the delivered values. + func testCurrentGenerationSyncProgressIsPublished() { + let manager = PlatformWalletManager() + + let currentGeneration = manager.shieldedSyncGeneration.current() + manager.handleShieldedSyncProgress( + cumulativeScanned: 4_096, + blockHeight: 200, + generation: currentGeneration + ) + + XCTAssertEqual( + manager.currentShieldedSyncScanned, + 4_096, + "a progress hop under the current generation must publish scanned" + ) + XCTAssertEqual( + manager.currentShieldedSyncBlockHeight, + 200, + "a progress hop under the current generation must publish block height" + ) + } + + // MARK: - Per-batch tree-progress guard + + /// A tree-progress hop captured under generation N is dropped once the + /// counter is bumped to N+1 before delivery: neither + /// `currentShieldedTreeCommitted` nor `currentShieldedTreeTotal` is set. + func testStaleTreeProgressIsDroppedAfterGenerationBump() { + let manager = PlatformWalletManager() + XCTAssertNil(manager.currentShieldedTreeCommitted, "fresh manager has no committed mirror") + XCTAssertNil(manager.currentShieldedTreeTotal, "fresh manager has no total mirror") + + // Snapshot the generation at "enqueue" time, then advance it (as + // stop/clear would) before delivery. + let capturedGeneration = manager.shieldedSyncGeneration.current() + manager.shieldedSyncGeneration.bump() + + manager.handleShieldedTreeProgress( + committed: 512, + total: 1_000, + generation: capturedGeneration + ) + + XCTAssertNil( + manager.currentShieldedTreeCommitted, + "a tree-progress hop under a superseded generation must not publish committed" + ) + XCTAssertNil( + manager.currentShieldedTreeTotal, + "a tree-progress hop under a superseded generation must not publish total" + ) + } + + /// A tree-progress hop captured under the CURRENT generation is published: + /// both `currentShieldedTree*` mirrors are set to the delivered values. + func testCurrentGenerationTreeProgressIsPublished() { + let manager = PlatformWalletManager() + + let currentGeneration = manager.shieldedSyncGeneration.current() + manager.handleShieldedTreeProgress( + committed: 768, + total: 2_000, + generation: currentGeneration + ) + + XCTAssertEqual( + manager.currentShieldedTreeCommitted, + 768, + "a tree-progress hop under the current generation must publish committed" + ) + XCTAssertEqual( + manager.currentShieldedTreeTotal, + 2_000, + "a tree-progress hop under the current generation must publish total" + ) + } +} diff --git a/packages/swift-sdk/SwiftExampleApp/docs/shielded-sync-timing-spec.md b/packages/swift-sdk/SwiftExampleApp/docs/shielded-sync-timing-spec.md new file mode 100644 index 00000000000..f4d89591de5 --- /dev/null +++ b/packages/swift-sdk/SwiftExampleApp/docs/shielded-sync-timing-spec.md @@ -0,0 +1,353 @@ +# Shielded sync timing — SwiftExampleApp spec (revised) + +## Goal + +When the user runs SwiftExampleApp against a devnet whose chain has a +pre-seeded shielded pool (N=1M notes via `dashpay/drive:3.1-shielded.2`), +**they need to see in the UI how long a shielded sync pass took, AND +have a live signal that an in-flight sync is making progress.** + +Primary use case: confirming initial wallet sync against a 1M-note +devnet completes within expected wall-clock (~10 min on M-series), and +that the user can tell from the UI whether sync is alive vs hung during +the long initial pass. + +## Why the goal expanded vs the first draft + +Adversarial review surfaced that a post-hoc-only duration is theatre at +N=1M: the user stares at a "Syncing..." spinner for ten minutes with no +signal "alive vs hung" — exactly the failure mode this exercise is +meant to detect. A live elapsed display is therefore mandatory for the +primary use case, not optional. + +## Scope (two surfaces) + +### Surface 1 — timing display (the primary task) + +The existing "ZK Shielded Sync Status" section in +`CoreContentView.swift` (≈ lines 391–510). One inline addition to that +section, two new `@Published` fields on `ShieldedService`, one console +log line per pass. No existing functionality changes. + +### Surface 2 — TEMPORARY test-wallet import (to make the timing meaningful) + +To validate against `dashpay/drive:3.1-shielded.2` and recover the +seeded 400 000 balance, the iOS app needs to sync wallet A whose +ZIP-32 seed is the **raw bytes `[0x73; 32]`** — not derivable from +any BIP-39 mnemonic. + +This requires: + +1. **New FFI** `platform_wallet_manager_bind_shielded_with_raw_seed` + in `packages/rs-platform-wallet-ffi/src/shielded_sync.rs` — sibling + to the existing `platform_wallet_manager_bind_shielded`, but + accepts a `seed_bytes: *const u8` + `seed_len: usize` instead of + a `MnemonicResolverHandle`. Calls + `wallet.bind_shielded(raw_seed.as_slice(), ...)` directly. +2. **Swift wrapper** `bindShieldedRawSeed(walletId:rawSeed:accounts:)` + in `Sources/SwiftDashSDK/PlatformWallet/...` +3. **Debug-only UI button** "Bind Test Wallet A (Shielded)" in the + existing ZK Shielded Sync Status section. Hardcodes `[0x73; 32]` + and calls the new wrapper. + +**Everything in Surface 2 is tagged with explicit removal TODOs.** +Example tag: +``` +// TODO(shielded-snapshot-devnet-test): remove this FFI entry once +// SwiftExampleApp adopts a proper test-wallet import flow. +// Tracked: dashpay/platform#3714. +``` + +Surface 2 lives behind no `#[cfg]` gate (would complicate the +release build), so the TODO comments are the contract: this surface +is provisional and removed before merging the ultimate version of +PR #3732. + +## Non-goals + +- Not building a separate sync dashboard. +- Not orchestrating sync from Swift — `walletManager.startShieldedSync` + already runs the loop; we observe. +- Not exposing per-cmx mid-pass progress (e.g. "412k/1M scanned"). That + needs a Rust→Swift signal we don't have today. Out of scope. +- Not measuring "whole catch-up cycle" duration as a single value (see + §"Known limitation: catch-up vs per-pass" below). +- Not persisting timing across app launches — display state only. +- Not building a `timeSinceBind` / "cumulative since bind" number — it + grows unbounded and answers no concrete question. + +## Existing surface (recap) + +**Service:** `ShieldedService` (`Core/Services/ShieldedService.swift`) + +Existing `@Published`: +- `isSyncing: Bool` +- `lastSyncTime: Date?` (when the most recent sync **completed**) +- `syncCountSinceLaunch: Int` +- `totalScanned: UInt64`, `totalNewNotes: UInt64`, `totalNewlySpent: UInt64` +- counters, balance, address fields + +**UI:** `CoreContentView.swift` "ZK Shielded Sync Status" section +already shows `ProgressView()` + "Syncing..." while in-flight, +"Last sync: " via `lastSyncTime`, cumulative counters, +balance + Notes Synced watermark, Sync Now / Clear buttons. + +**Underlying flow (Rust → Swift):** +1. `walletManager.$shieldedSyncIsSyncing` publishes `Bool`. Flips true + at sync-pass start, false at completion. +2. `walletManager.$lastShieldedSyncEvent` publishes a + `ShieldedSyncEvent` on each pass completion. + +**Gap:** UI shows COUNTERS and relative completion time, but no +wall-clock duration of a single pass, and no live indication during a +10-minute initial sync. + +## Spec + +### S1. Service-level fields + +Two new `@Published` fields on `ShieldedService` (read by the UI): + +| Field | Type | Semantic | +|---|---|---| +| `lastSyncDuration: TimeInterval?` | seconds | wall-clock of the most recent non-cooldown sync pass (set at completion) | +| `currentSyncElapsed: TimeInterval?` | seconds | running wall-clock of the in-flight sync; ticks while `isSyncing == true`, nil otherwise | + +One new private field: + +| Field | Set when | +|---|---| +| `currentSyncStartedAt: Date?` | `isSyncing` Swift mirror transitions false → true | + +Both `@Published` fields stay nil until they have a real value to +show. Both reset to nil on `bind()` / `reset()` / `clearLocalState`. + +**No `lastBindCompletedAt`, no `lastSyncCompletedAt`, no `timeSinceBind`** — +dropped per Scope reviewer. + +### S2. Pass boundaries — Swift edges only + +Both pass endpoints are observed from the **Swift mirror of +`$shieldedSyncIsSyncing`**, not from Rust event timestamps. + +- **Start:** false → true transition of `isSyncing`. +- **End:** true → false transition of `isSyncing`. + +Rationale: `event.syncUnixSeconds` is integer-second resolution; mixing +it with `Date()` on the Swift edge can render negative or grossly +inflated durations for sub-second steady-state passes. Using the Swift +edge for both endpoints means the Rust↔Swift latency cancels out +symmetrically. + +### S3. Live ticker + +A 1-second `Timer` lives on the `ShieldedService` and: + +- Starts on the false → true transition. +- Tick handler: updates `currentSyncElapsed = Date() − currentSyncStartedAt`. +- Stops + nils `currentSyncElapsed` on the true → false transition. + +One timer source on the service rather than a per-view source — the +view subscribes to `$currentSyncElapsed` like any other published +field. Service is `@MainActor` so timer fires on main thread already. + +### S4. Edge handling (bug-fixes from review) + +- **B1 (clock skew):** Solved by S2 — Swift-edge for both endpoints. +- **B2 (failure leaves start stamped):** `currentSyncStartedAt` is + cleared on EVERY true → false transition, regardless of whether the + emitted `ShieldedSyncEvent` reports success or failure. Same for the + ticker. +- **B3 (`switchTo(walletId:)` silently resets timing):** Documented in + `bind()`'s comment block. UI behaviour after a wallet switch is the + same as a fresh bind — the row reappears after the next pass. +- **B4 (negative or absurd duration):** Clamp to `max(0, …)` in the + view formatter; if `currentSyncStartedAt` is nil at completion (which + shouldn't happen with S2 but is defensible defence-in-depth), set + `lastSyncDuration = nil` and skip the log line. +- **B5 (re-fire of `true`):** Set `currentSyncStartedAt` only when + transitioning **from** `false`. Track the previous mirror value + inside the `.sink` to detect the edge — the existing + `syncStateCancellable` `.sink` is the right place. +- **Cooldown skip:** `result.cooldownSkip == true` events do NOT update + `lastSyncDuration` (zeros are not signal). The ticker stops on the + edge regardless. No noisy log spam: emit the cooldown-skip log only + on the first one in a row; suppress contiguous skips. + +### S5. Console log + +In `handleShieldedSyncEvent`, when `result.success && !result.cooldownSkip`, +emit one line via `SDKLogger.log(.medium)`: + +``` +Shielded sync done pass= elapsed= rate= scanned= new= spent= balance= +``` + +- `.medium` (not `.high`) so devnet operators on default presets see + these without changing log settings. +- `rate` is suppressed when `elapsed ≈ 0` or `scanned == 0`. +- Format kept stable for `xcrun simctl spawn log` scraping. + +Skipped (cooldown) passes log a single `"Shielded sync skipped (cooldown)"` +at `.medium`; contiguous skips suppressed. + +Started passes log `"Shielded sync started"` at `.medium` on the +false → true edge. Paired with the "done" line for offline analysis. + +### S6. UI + +Inside the existing "ZK Shielded Sync Status" section, immediately +under the existing "Queries Since Launch" row (and above the badges): + +**While `isSyncing` is true AND `currentSyncElapsed != nil`:** +``` +Syncing… elapsed: 4.2 s +``` + +**While `isSyncing` is false AND `lastSyncDuration != nil`:** +``` +Last sync duration: 12.4 s +``` + +**Otherwise:** no row (clean state pre-first-sync, matches existing +"Not synced yet" behaviour). + +Layout: same `HStack { Text(label) ; Spacer() ; Text(value).monospacedDigit() }` +pattern as the surrounding rows. Mono digits keep the number readable +as it ticks. + +The existing `ProgressView()` + "Syncing..." spinner stays unchanged — +it's the qualitative "is something happening" affordance; the new row +is the quantitative "how long has it been going" affordance. + +### S7. Reset sites + +All three private/published fields nil'd in: +- `bind()` — before the new bind runs (so post-bind sync gets a clean + baseline, not stale from a prior wallet). +- `reset()` — full teardown. +- `clearLocalState` — global clear. +- True → false edge — both `currentSyncStartedAt` and + `currentSyncElapsed` nil'd (S4). + +### S8. What this does NOT change + +- No new FFI signatures. +- No changes to `rs-platform-wallet-ffi` or `rs-platform-wallet`. +- No changes to `PlatformWalletManager`. +- No SwiftData schema change. +- No new view / screen / sheet / menu entry. + +## Known limitation: catch-up vs per-pass + +At N=1M, the manager loop runs MANY internal passes during initial +catch-up — each is one `ShieldedSyncEvent`. We measure **per-pass** +wall-clock. The user-facing reality is that for an initial catch-up +the "Last sync duration" they see at the end will be the **last +pass**'s time (likely a few seconds — the trailing partial chunk), +NOT the whole catch-up's 10 minutes. + +This is acceptable for the live use case (the live ticker covers the +"is it alive" question), but means the post-hoc number reads smaller +than the user's perceived wall-clock for the first catch-up. The +**console log** mitigates by recording every pass — sum from the log +to get total catch-up time. + +A proper "catch-up completed" signal would require either: +- A Rust-side signal `next_start_index == tree_size` exposed as a + derived `isCaughtUp` event, OR +- Synthesizing it Swift-side from `event.totalScanned` + a tree-size + read. + +Both expand the scope materially. Deferred. The current per-pass +number + live ticker covers the primary use case ("is it alive, how +long is each pass taking"). + +## Architecture conformance (`swift-sdk/CLAUDE.md`) + +- ✅ Persist / load / bridge only. Timing fields are display state + derived from existing Combine publishers — no business logic, no + decisions, no orchestration. +- ✅ No new FFI surface. +- ✅ Timer is a UI-driving mechanism, not a policy loop. The decision + to keep syncing lives on the Rust side; the timer just animates + `currentSyncElapsed` for the view. + +## Test plan + +1. **Smoke (local dashmate devnet):** + - Bind a wallet; observe one sync pass. + - UI shows "Syncing… elapsed: X.Xs" with X ticking visibly. + - On completion: UI shows "Last sync duration: Y.Ys" with Y a + reasonable positive value. + - Console: paired "started" + "done" log lines at `.medium`. + +2. **Reset / clear behaviour:** + - After a successful sync, hit "Clear". UI row disappears. + - Hit "Sync Now". Row reappears live, then settles to post-hoc. + +3. **Cooldown skip:** + - After steady-state, force a cooldown-skip pass. UI does not + update `lastSyncDuration` (stays at the prior value). Console + shows one "skipped (cooldown)" line and no further skips until + the next real pass. + +4. **Failure path (offline):** + - Disconnect the gateway, hit Sync Now. Confirm `lastSyncDuration` + is NOT updated by the failed pass. `currentSyncStartedAt` IS + cleared (verified indirectly: next successful pass shows correct + duration, not absurdly inflated). + +5. **N=1M devnet (validation):** point the app at + `dashpay/drive:3.1-shielded.2` running on devnet. Bind a fresh + wallet: + - Live ticker increments visibly through the long initial pass. + - At the end, console log lines sum to the user-perceived wall-clock. + - Each subsequent pass shows a small (seconds) duration. + +## File touch list + +### Surface 1 (timing, permanent) + +- `packages/swift-sdk/SwiftExampleApp/SwiftExampleApp/Core/Services/ShieldedService.swift` + - +1 private field (`currentSyncStartedAt`), +2 `@Published` + fields (`lastSyncDuration`, `currentSyncElapsed`). + - +1 Timer (`syncTickTimer: Timer?`). + - +1 previous-mirror state on the existing `syncStateCancellable.sink` + to detect false → true edges. + - Reset sites (`bind`, `reset`, `clearLocalState`) get the new + fields nil'd. + - `handleShieldedSyncEvent` logs at `.medium` on success path; + cooldown-skip path emits one suppressed log + leaves duration + alone. +- `packages/swift-sdk/SwiftExampleApp/SwiftExampleApp/Core/Views/CoreContentView.swift` + - +1 inline row in the existing "ZK Shielded Sync Status" section + (≈ line 439 area, alongside "Queries Since Launch"). + +### Surface 2 (raw-seed test wallet bind, TEMPORARY) + +All marked with the same removal TODO tag. + +- `packages/rs-platform-wallet-ffi/src/shielded_sync.rs` + - +1 new FFI entry + `platform_wallet_manager_bind_shielded_with_raw_seed` + alongside the existing `platform_wallet_manager_bind_shielded`. + Same parameters EXCEPT replaces `mnemonic_resolver_handle` with + `seed_bytes: *const u8 + seed_len: usize`. +- `packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerShieldedBind.swift` (or wherever `bindShielded` lives) + - +1 thin wrapper method + `bindShieldedRawSeed(walletId:rawSeed:accounts:)` that calls + the new FFI. +- `packages/swift-sdk/SwiftExampleApp/SwiftExampleApp/Core/Views/CoreContentView.swift` + - +1 button "Bind Test Wallet A (Shielded)" inside the ZK + Shielded Sync Status section. Hardcodes `[0x73; 32]` and calls + the new wrapper. Button is visible only when the active wallet + has no shielded binding (matches the existing "Sync Now" + affordance's gating). + +### Diff size estimate + +- Surface 1 (timing): ~60 lines additive. +- Surface 2 (raw-seed test wallet): ~70 lines additive. +- Total: ~130 lines, no removals, no API drift to existing + functionality. diff --git a/packages/wasm-sdk/src/queries/shielded.rs b/packages/wasm-sdk/src/queries/shielded.rs index 57355087bdd..dd2c3cc2acf 100644 --- a/packages/wasm-sdk/src/queries/shielded.rs +++ b/packages/wasm-sdk/src/queries/shielded.rs @@ -93,7 +93,7 @@ impl WasmSdk { let array = Array::new(); if let Some(notes) = result { - for note in notes.0 { + for note in notes.notes { array.push(&JsValue::from(ShieldedEncryptedNoteWasm { cmx: note.cmx, nullifier: note.nullifier, @@ -213,7 +213,7 @@ impl WasmSdk { let array = Array::new(); if let Some(notes) = result { - for note in notes.0 { + for note in notes.notes { array.push(&JsValue::from(ShieldedEncryptedNoteWasm { cmx: note.cmx, nullifier: note.nullifier, diff --git a/scripts/setup_local_network.sh b/scripts/setup_local_network.sh index 543262b3ab2..4eaafcadf65 100755 --- a/scripts/setup_local_network.sh +++ b/scripts/setup_local_network.sh @@ -16,3 +16,22 @@ yarn run dashmate setup local --verbose \ # enable insight yarn dashmate config set core.insight.enabled true --config local_seed + +# Bake SDK_TEST_DATA=true into the drive-abci docker build for each masternode +# so the genesis shielded-pool seeder + identity/contract fixtures run on every +# local devnet bring-up. Production / release builds explicitly do NOT set this. +# +# CARGO_BUILD_PROFILE=release is mandatory at the current default N=1_000_000. +# Release-mode Sinsemilla is ~10× faster than debug; without it the bake +# stage during `docker build` would take hours. Apply at runtime InitChain +# is ~134 ms regardless of profile (single SST ingest, no Sinsemilla work). +# +# Drop back to `dev` only if you also lower +# `ShieldedSeedConfig::sdk_test_data().total_notes` to ~5k or below +# (debug bake fits in tenderdash's InitChain window only at small N). +for i in $(seq 1 ${MASTERNODES_COUNT}); do + yarn dashmate config set --config=local_${i} \ + platform.drive.abci.docker.build.buildArgs.SDK_TEST_DATA '"true"' + yarn dashmate config set --config=local_${i} \ + platform.drive.abci.docker.build.buildArgs.CARGO_BUILD_PROFILE '"release"' +done