diff --git a/Cargo.lock b/Cargo.lock index 673af9d..ff4608f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1676,7 +1676,7 @@ dependencies = [ [[package]] name = "fula-api" -version = "0.6.2" +version = "0.6.3" dependencies = [ "anyhow", "axum", @@ -1705,7 +1705,7 @@ dependencies = [ [[package]] name = "fula-blockstore" -version = "0.6.2" +version = "0.6.3" dependencies = [ "anyhow", "async-trait", @@ -1743,7 +1743,7 @@ dependencies = [ [[package]] name = "fula-cli" -version = "0.6.2" +version = "0.6.3" dependencies = [ "anyhow", "async-trait", @@ -1795,7 +1795,7 @@ dependencies = [ [[package]] name = "fula-client" -version = "0.6.2" +version = "0.6.3" dependencies = [ "anyhow", "async-trait", @@ -1837,7 +1837,7 @@ dependencies = [ [[package]] name = "fula-core" -version = "0.6.2" +version = "0.6.3" dependencies = [ "anyhow", "async-trait", @@ -1872,7 +1872,7 @@ dependencies = [ [[package]] name = "fula-crypto" -version = "0.6.2" +version = "0.6.3" dependencies = [ "aes-gcm", "anyhow", @@ -1917,7 +1917,7 @@ dependencies = [ [[package]] name = "fula-flutter" -version = "0.6.2" +version = "0.6.3" dependencies = [ "anyhow", "async-lock", @@ -1940,7 +1940,7 @@ dependencies = [ [[package]] name = "fula-js" -version = "0.6.2" +version = "0.6.3" dependencies = [ "base64 0.22.1", "bytes", diff --git a/Cargo.toml b/Cargo.toml index b061633..82c9db4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,7 +79,7 @@ name = "encrypted_upload_test" path = "examples/encrypted_upload_test.rs" [workspace.package] -version = "0.6.2" +version = "0.6.3" edition = "2021" license = "MIT OR Apache-2.0" repository = "https://github.com/functionland/fula-api" diff --git a/crates/fula-client/src/client.rs b/crates/fula-client/src/client.rs index 91e0093..dc88656 100644 --- a/crates/fula-client/src/client.rs +++ b/crates/fula-client/src/client.rs @@ -1012,6 +1012,62 @@ impl FulaClient { bucket: &str, key: &str, cid_hint: &cid::Cid, + ) -> Result { + // Generic cid-hint entry point. A 404/NoSuchKey from a reachable + // master propagates UNCHANGED (a missing object is a real miss, + // not an offline condition); only master-unreachable consults the + // hint. This strict invariant is covered by + // `test_cid_hint_master_4xx_propagates_without_fallback` — do not + // weaken it. Forest infrastructure that must recover a gc-orphaned + // node on a 404 uses `get_forest_object_known_cid` instead. + self.get_object_with_offline_fallback_known_cid_inner(bucket, key, cid_hint, false) + .await + } + + /// Forest-infrastructure variant of + /// [`get_object_with_offline_fallback_known_cid`] (issue #24). + /// + /// Identical to the generic method EXCEPT it ALSO engages the verified + /// gateway race when a reachable master returns a 404/NoSuchKey — + /// recovering a forest node or manifest page whose gateway + /// storage-key → CID index entry was destroyed by a server-side + /// `ipfs repo gc` while the block itself still exists in IPFS by CID. + /// + /// Scoped to forest infrastructure (HAMT nodes + manifest pages) on + /// purpose: those callers hold an authoritative, freshly-decrypted CID + /// from the parent `LinkV2` / `page_ref`, so a 404 there means "index + /// entry gc'd", not "object deleted". The fetched bytes are + /// content-verified against the supplied CID inside the gateway race + /// (`fetch_verified` → `verify_cid_against_bytes`), so recovery can + /// only ever return the exact block the manifest points at — the CID + /// is the capability. The generic method keeps the strict + /// propagate-404 invariant so no future non-forest caller can silently + /// inherit hide-404 behavior. + #[cfg(not(target_arch = "wasm32"))] + #[instrument(skip(self, cid_hint))] + pub async fn get_forest_object_known_cid( + &self, + bucket: &str, + key: &str, + cid_hint: &cid::Cid, + ) -> Result { + self.get_object_with_offline_fallback_known_cid_inner(bucket, key, cid_hint, true) + .await + } + + /// Shared implementation of the two cid-hint entry points above. + /// `recover_on_not_found` gates the issue-#24 behavior: `true` routes + /// a 404/NoSuchKey from a reachable master into the verified gateway + /// race (forest nodes/pages); `false` propagates the 404 unchanged + /// (generic objects). The master-up success path and the + /// master-unreachable branch are identical for both. + #[cfg(not(target_arch = "wasm32"))] + async fn get_object_with_offline_fallback_known_cid_inner( + &self, + bucket: &str, + key: &str, + cid_hint: &cid::Cid, + recover_on_not_found: bool, ) -> Result { // Fast path mirror of get_object_with_offline_fallback — flags // off means no hint consumption, identical latency. @@ -1057,7 +1113,37 @@ impl FulaClient { freshness: ReadFreshness::Live, }) } - Err(e) if is_master_unreachable_error(&e) => { + // Issue #24: forest infrastructure (`recover_on_not_found`) ALSO + // recovers on a NoSuchKey/404 from a REACHABLE master — the + // node/page's gateway storage-key→CID index entry was gc-orphaned + // while the block still exists by CID. We match the shared + // `is_not_found()` (which also covers NoSuchBucket) rather than a + // narrower object-only predicate on purpose: the two forest + // callers (`S3BlobBackend::get_with_cid_hint`, `load_manifest_pages`) + // are reached only AFTER the bucket's manifest decrypted, so a + // NoSuchBucket cannot occur here, and the gateway race is + // content-verified against the manifest-supplied CID regardless of + // which not-found shape triggered it (caller scope verified at + // source during review). + Err(e) + if is_master_unreachable_error(&e) + || (recover_on_not_found && e.is_not_found()) => + { + // Surface gc-orphan recovery: a silent recover would hide + // ongoing gateway-index rot. Fires only on the reachable- + // master not-found path, not the normal master-unreachable + // case (which is the expected offline-mode flow). + if recover_on_not_found && !is_master_unreachable_error(&e) { + warn!( + bucket = %bucket, + key = %key, + cid = %cid_hint, + "forest node/page not-found from reachable master — \ + gateway index entry likely gc-orphaned; recovering via \ + verified gateway race by CID (issue #24). Repeated \ + occurrences mean the gateway index needs re-pin." + ); + } self.try_offline_fallback_with_cid_hint(bucket, key, cid_hint, e) .await } diff --git a/crates/fula-client/src/encryption.rs b/crates/fula-client/src/encryption.rs index c20ce15..562a9cb 100644 --- a/crates/fula-client/src/encryption.rs +++ b/crates/fula-client/src/encryption.rs @@ -517,7 +517,7 @@ impl BlobBackend for S3BlobBackend { attempt += 1; match self .inner - .get_object_with_offline_fallback_known_cid(&self.bucket, path, cid) + .get_forest_object_known_cid(&self.bucket, path, cid) .await { Ok(result) => return Ok(result.inner.data.to_vec()), @@ -5279,7 +5279,7 @@ impl EncryptedClient { let (blob, observed_etag) = match cid_hint { Some(cid) => self .inner - .get_object_with_offline_fallback_known_cid(bucket, &page_key, &cid) + .get_forest_object_known_cid(bucket, &page_key, &cid) .await, None => { self.inner diff --git a/crates/fula-client/tests/walkable_v8_offline_walk.rs b/crates/fula-client/tests/walkable_v8_offline_walk.rs index a2a300f..1210d25 100644 --- a/crates/fula-client/tests/walkable_v8_offline_walk.rs +++ b/crates/fula-client/tests/walkable_v8_offline_walk.rs @@ -195,3 +195,65 @@ async fn s3_backend_get_with_cid_hint_rejects_gateway_tamper() { walkable-v8 inherits from gateway_fetch." ); } + +/// Online orphaned-node recovery (issue #24). Master is UP but returns +/// `404 NoSuchKey` for a forest node whose storage-key -> CID index entry +/// was destroyed by a server-side `ipfs repo gc`, while the node's block +/// still exists in IPFS by CID. The forest cid-hint fetch MUST recover it +/// via the verified gateway race (the same mechanism the master-down path +/// already uses), not abort the walk. +/// +/// Pre-fix this FAILS: `get_with_cid_hint` routes through +/// `get_object_with_offline_fallback_known_cid`, which only engages the +/// race on `is_master_unreachable_error`; a 404 is not master-down, so it +/// propagates and the `.expect()` below panics. Post-fix the forest-scoped +/// wrapper also recovers on a `NotFound`, so the node resolves by CID. +#[tokio::test] +async fn s3_backend_get_with_cid_hint_recovers_orphaned_node_on_master_404() { + let master = MockServer::start().await; + let gateway = MockServer::start().await; + + let body = b"opaque-v8-hamt-node-ciphertext-orphaned-by-gc".to_vec(); + let body_cid = blake3_raw_cid(&body); + + // Master is UP but the storage-key -> CID index entry is gone: + // returns 404 NoSuchKey (NOT a master-down 5xx). + Mock::given(method("GET")) + .and(wm_path("/images/__fula_forest_v7_nodes/orphaned")) + .respond_with( + ResponseTemplate::new(404).set_body_string( + r#"NoSuchKeynot here"#, + ), + ) + .mount(&master) + .await; + + // The node's block still exists in IPFS, addressable by CID. + let gateway_path = format!("/ipfs/{}", body_cid); + Mock::given(method("GET")) + .and(wm_path(gateway_path.as_str())) + .respond_with(ResponseTemplate::new(200).set_body_bytes(body.clone())) + .mount(&gateway) + .await; + + let gateway_template = format!("{}/ipfs/{{cid}}", gateway.uri()); + let config = mk_config_with_gateway(&master.uri(), &gateway_template); + let client = FulaClient::new(config).expect("build FulaClient"); + let backend = S3BlobBackend::new(client, "images".to_string()); + + let result = backend + .get_with_cid_hint("__fula_forest_v7_nodes/orphaned", Some(&body_cid)) + .await + .expect( + "issue #24: a forest node that 404s by storage-key but exists by \ + CID must be recovered via the verified gateway race even with \ + master UP. Pre-fix this propagates the 404 and aborts the walk.", + ); + + assert_eq!( + result, body, + "recovered bytes must content-address to the manifest-supplied CID; \ + a successful Ok confirms the gateway race executed and passed \ + verify_cid_against_bytes", + ); +} diff --git a/crates/fula-js/Cargo.toml b/crates/fula-js/Cargo.toml index 35906a4..ad40eaf 100644 --- a/crates/fula-js/Cargo.toml +++ b/crates/fula-js/Cargo.toml @@ -4,7 +4,7 @@ description = "JavaScript/TypeScript SDK for Fula decentralized storage - WASM b # Hard-coded (not workspace-inherited) because wasm-pack <= 0.13 fails # to parse `*.workspace = true` keys in its own manifest scan. Keep # these in sync with `[workspace.package]` in the root Cargo.toml. -version = "0.6.2" +version = "0.6.3" edition = "2021" license = "MIT OR Apache-2.0" repository = "https://github.com/functionland/fula-api" diff --git a/packages/fula_client/CHANGELOG.md b/packages/fula_client/CHANGELOG.md index eb43381..58fa5d1 100644 --- a/packages/fula_client/CHANGELOG.md +++ b/packages/fula_client/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.6.2] - 2026-05-23 +### Fixed +- **#24 — Fixed 404 breaks.** + +## [0.6.2] - 2026-05-23 + ### Fixed - **#23 — `uploadLargeFileResumable` now durably persists the forest to master.** Previously, the resumable chunked upload path (`put_object_encrypted_resumable_with_cancel` and `resume_upload_with_cancel`) registered the new entry only in the in-memory cache + on-disk WAL after chunks landed; the master-side flush sequence (`save_sharded_hamt_forest`/`save_monolithic_forest` → `Persisting bucket registry to IPFS` → `Bucket root CID enqueued for durable pin (W.9.6)` → IPNS update) never fired. A fresh client load — post app restart, post storage clear, fresh device — pulled the pre-upload forest and the file was missing despite the upload returning a valid etag. Symmetric with `put_object_flat`'s established "register + flush" pattern; the resumable path was added in v0.6.0 (issues #16/#17/#18) but the flush step was inadvertently omitted. Fix is one line inside `finalize_and_register_resumed_upload` (`crates/fula-client/src/encryption.rs`): call `self.flush_forest_locked(bucket)` after `register_encrypted_chunked_upload_in_forest`, before manifest deletion. Uses the `_locked` variant because both callers already hold `bucket_write_mutex` (issue #16); `flush_forest`'s public variant would deadlock on tokio's non-reentrant mutex. Manifest deletion moved after the flush so crash-safety extends to flush failure (manifest stays on disk for `resume_upload` retry). diff --git a/packages/fula_client/ios/fula_client.podspec b/packages/fula_client/ios/fula_client.podspec index 85d07de..e0953bf 100644 --- a/packages/fula_client/ios/fula_client.podspec +++ b/packages/fula_client/ios/fula_client.podspec @@ -6,7 +6,7 @@ Pod::Spec.new do |s| s.name = 'fula_client' - s.version = '0.6.2' + s.version = '0.6.3' s.summary = 'Flutter SDK for Fula decentralized storage' s.description = <<-DESC A Flutter plugin providing client-side encryption, metadata privacy, diff --git a/packages/fula_client/pubspec.yaml b/packages/fula_client/pubspec.yaml index 003dba5..21a16f1 100644 --- a/packages/fula_client/pubspec.yaml +++ b/packages/fula_client/pubspec.yaml @@ -1,6 +1,6 @@ name: fula_client description: Flutter SDK for Fula decentralized storage with client-side encryption, metadata privacy, and secure sharing. -version: 0.6.2 +version: 0.6.3 homepage: https://fx.land repository: https://github.com/functionland/fula-api issue_tracker: https://github.com/functionland/fula-api/issues