fix(client): recover gc-orphaned forest nodes on 404 via verified CID race

Cargo.toml

@@ -79,7 +79,7 @@ name = "encrypted_upload_test"
 path = "examples/encrypted_upload_test.rs"
 
 [workspace.package]
-version = "0.6.2"
+version = "0.6.3"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/functionland/fula-api"

crates/fula-client/src/client.rs

@@ -1012,6 +1012,62 @@ impl FulaClient {
         bucket: &str,
         key: &str,
         cid_hint: &cid::Cid,
+    ) -> Result<OfflineGetResult> {
+        // Generic cid-hint entry point. A 404/NoSuchKey from a reachable
+        // master propagates UNCHANGED (a missing object is a real miss,
+        // not an offline condition); only master-unreachable consults the
+        // hint. This strict invariant is covered by
+        // `test_cid_hint_master_4xx_propagates_without_fallback` — do not
+        // weaken it. Forest infrastructure that must recover a gc-orphaned
+        // node on a 404 uses `get_forest_object_known_cid` instead.
+        self.get_object_with_offline_fallback_known_cid_inner(bucket, key, cid_hint, false)
+            .await
+    }
+
+    /// Forest-infrastructure variant of
+    /// [`get_object_with_offline_fallback_known_cid`] (issue #24).
+    ///
+    /// Identical to the generic method EXCEPT it ALSO engages the verified
+    /// gateway race when a reachable master returns a 404/NoSuchKey —
+    /// recovering a forest node or manifest page whose gateway
+    /// storage-key → CID index entry was destroyed by a server-side
+    /// `ipfs repo gc` while the block itself still exists in IPFS by CID.
+    ///
+    /// Scoped to forest infrastructure (HAMT nodes + manifest pages) on
+    /// purpose: those callers hold an authoritative, freshly-decrypted CID
+    /// from the parent `LinkV2` / `page_ref`, so a 404 there means "index
+    /// entry gc'd", not "object deleted". The fetched bytes are
+    /// content-verified against the supplied CID inside the gateway race
+    /// (`fetch_verified` → `verify_cid_against_bytes`), so recovery can
+    /// only ever return the exact block the manifest points at — the CID
+    /// is the capability. The generic method keeps the strict
+    /// propagate-404 invariant so no future non-forest caller can silently
+    /// inherit hide-404 behavior.
+    #[cfg(not(target_arch = "wasm32"))]
+    #[instrument(skip(self, cid_hint))]
+    pub async fn get_forest_object_known_cid(
+        &self,
+        bucket: &str,
+        key: &str,
+        cid_hint: &cid::Cid,
+    ) -> Result<OfflineGetResult> {
+        self.get_object_with_offline_fallback_known_cid_inner(bucket, key, cid_hint, true)
+            .await
+    }
+
+    /// Shared implementation of the two cid-hint entry points above.
+    /// `recover_on_not_found` gates the issue-#24 behavior: `true` routes
+    /// a 404/NoSuchKey from a reachable master into the verified gateway
+    /// race (forest nodes/pages); `false` propagates the 404 unchanged
+    /// (generic objects). The master-up success path and the
+    /// master-unreachable branch are identical for both.
+    #[cfg(not(target_arch = "wasm32"))]
+    async fn get_object_with_offline_fallback_known_cid_inner(
+        &self,
+        bucket: &str,
+        key: &str,
+        cid_hint: &cid::Cid,
+        recover_on_not_found: bool,
     ) -> Result<OfflineGetResult> {
         // Fast path mirror of get_object_with_offline_fallback — flags
         // off means no hint consumption, identical latency.
@@ -1057,7 +1113,37 @@ impl FulaClient {
                     freshness: ReadFreshness::Live,
                 })
             }
-            Err(e) if is_master_unreachable_error(&e) => {
+            // Issue #24: forest infrastructure (`recover_on_not_found`) ALSO
+            // recovers on a NoSuchKey/404 from a REACHABLE master — the
+            // node/page's gateway storage-key→CID index entry was gc-orphaned
+            // while the block still exists by CID. We match the shared
+            // `is_not_found()` (which also covers NoSuchBucket) rather than a
+            // narrower object-only predicate on purpose: the two forest
+            // callers (`S3BlobBackend::get_with_cid_hint`, `load_manifest_pages`)
+            // are reached only AFTER the bucket's manifest decrypted, so a
+            // NoSuchBucket cannot occur here, and the gateway race is
+            // content-verified against the manifest-supplied CID regardless of
+            // which not-found shape triggered it (caller scope verified at
+            // source during review).
+            Err(e)
+                if is_master_unreachable_error(&e)
+                    || (recover_on_not_found && e.is_not_found()) =>
+            {
+                // Surface gc-orphan recovery: a silent recover would hide
+                // ongoing gateway-index rot. Fires only on the reachable-
+                // master not-found path, not the normal master-unreachable
+                // case (which is the expected offline-mode flow).
+                if recover_on_not_found && !is_master_unreachable_error(&e) {
+                    warn!(
+                        bucket = %bucket,
+                        key = %key,
+                        cid = %cid_hint,
+                        "forest node/page not-found from reachable master — \
+                         gateway index entry likely gc-orphaned; recovering via \
+                         verified gateway race by CID (issue #24). Repeated \
+                         occurrences mean the gateway index needs re-pin."
+                    );
+                }
                 self.try_offline_fallback_with_cid_hint(bucket, key, cid_hint, e)
                     .await
             }

crates/fula-client/src/encryption.rs

@@ -517,7 +517,7 @@ impl BlobBackend for S3BlobBackend {
             attempt += 1;
             match self
                 .inner
-                .get_object_with_offline_fallback_known_cid(&self.bucket, path, cid)
+                .get_forest_object_known_cid(&self.bucket, path, cid)
                 .await
             {
                 Ok(result) => return Ok(result.inner.data.to_vec()),
@@ -5279,7 +5279,7 @@ impl EncryptedClient {
             let (blob, observed_etag) = match cid_hint {
                 Some(cid) => self
                     .inner
-                    .get_object_with_offline_fallback_known_cid(bucket, &page_key, &cid)
+                    .get_forest_object_known_cid(bucket, &page_key, &cid)
                     .await,
                 None => {
                     self.inner

crates/fula-client/tests/walkable_v8_offline_walk.rs

@@ -195,3 +195,65 @@ async fn s3_backend_get_with_cid_hint_rejects_gateway_tamper() {
          walkable-v8 inherits from gateway_fetch."
     );
 }
+
+/// Online orphaned-node recovery (issue #24). Master is UP but returns
+/// `404 NoSuchKey` for a forest node whose storage-key -> CID index entry
+/// was destroyed by a server-side `ipfs repo gc`, while the node's block
+/// still exists in IPFS by CID. The forest cid-hint fetch MUST recover it
+/// via the verified gateway race (the same mechanism the master-down path
+/// already uses), not abort the walk.
+///
+/// Pre-fix this FAILS: `get_with_cid_hint` routes through
+/// `get_object_with_offline_fallback_known_cid`, which only engages the
+/// race on `is_master_unreachable_error`; a 404 is not master-down, so it
+/// propagates and the `.expect()` below panics. Post-fix the forest-scoped
+/// wrapper also recovers on a `NotFound`, so the node resolves by CID.
+#[tokio::test]
+async fn s3_backend_get_with_cid_hint_recovers_orphaned_node_on_master_404() {
+    let master = MockServer::start().await;
+    let gateway = MockServer::start().await;
+
+    let body = b"opaque-v8-hamt-node-ciphertext-orphaned-by-gc".to_vec();
+    let body_cid = blake3_raw_cid(&body);
+
+    // Master is UP but the storage-key -> CID index entry is gone:
+    // returns 404 NoSuchKey (NOT a master-down 5xx).
+    Mock::given(method("GET"))
+        .and(wm_path("/images/__fula_forest_v7_nodes/orphaned"))
+        .respond_with(
+            ResponseTemplate::new(404).set_body_string(
+                r#"<Error><Code>NoSuchKey</Code><Message>not here</Message></Error>"#,
+            ),
+        )
+        .mount(&master)
+        .await;
+
+    // The node's block still exists in IPFS, addressable by CID.
+    let gateway_path = format!("/ipfs/{}", body_cid);
+    Mock::given(method("GET"))
+        .and(wm_path(gateway_path.as_str()))
+        .respond_with(ResponseTemplate::new(200).set_body_bytes(body.clone()))
+        .mount(&gateway)
+        .await;
+
+    let gateway_template = format!("{}/ipfs/{{cid}}", gateway.uri());
+    let config = mk_config_with_gateway(&master.uri(), &gateway_template);
+    let client = FulaClient::new(config).expect("build FulaClient");
+    let backend = S3BlobBackend::new(client, "images".to_string());
+
+    let result = backend
+        .get_with_cid_hint("__fula_forest_v7_nodes/orphaned", Some(&body_cid))
+        .await
+        .expect(
+            "issue #24: a forest node that 404s by storage-key but exists by \
+             CID must be recovered via the verified gateway race even with \
+             master UP. Pre-fix this propagates the 404 and aborts the walk.",
+        );
+
+    assert_eq!(
+        result, body,
+        "recovered bytes must content-address to the manifest-supplied CID; \
+         a successful Ok confirms the gateway race executed and passed \
+         verify_cid_against_bytes",
+    );
+}

crates/fula-js/Cargo.toml

@@ -4,7 +4,7 @@ description = "JavaScript/TypeScript SDK for Fula decentralized storage - WASM b
 # Hard-coded (not workspace-inherited) because wasm-pack <= 0.13 fails
 # to parse `*.workspace = true` keys in its own manifest scan. Keep
 # these in sync with `[workspace.package]` in the root Cargo.toml.
-version = "0.6.2"
+version = "0.6.3"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/functionland/fula-api"

packages/fula_client/CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [0.6.2] - 2026-05-23
 
+### Fixed
+- **#24 — Fixed 404 breaks.**
+
+## [0.6.2] - 2026-05-23
+
 ### Fixed
 
 - **#23 — `uploadLargeFileResumable` now durably persists the forest to master.** Previously, the resumable chunked upload path (`put_object_encrypted_resumable_with_cancel` and `resume_upload_with_cancel`) registered the new entry only in the in-memory cache + on-disk WAL after chunks landed; the master-side flush sequence (`save_sharded_hamt_forest`/`save_monolithic_forest` → `Persisting bucket registry to IPFS` → `Bucket root CID enqueued for durable pin (W.9.6)` → IPNS update) never fired. A fresh client load — post app restart, post storage clear, fresh device — pulled the pre-upload forest and the file was missing despite the upload returning a valid etag. Symmetric with `put_object_flat`'s established "register + flush" pattern; the resumable path was added in v0.6.0 (issues #16/#17/#18) but the flush step was inadvertently omitted. Fix is one line inside `finalize_and_register_resumed_upload` (`crates/fula-client/src/encryption.rs`): call `self.flush_forest_locked(bucket)` after `register_encrypted_chunked_upload_in_forest`, before manifest deletion. Uses the `_locked` variant because both callers already hold `bucket_write_mutex` (issue #16); `flush_forest`'s public variant would deadlock on tokio's non-reentrant mutex. Manifest deletion moved after the flush so crash-safety extends to flush failure (manifest stays on disk for `resume_upload` retry).

packages/fula_client/ios/fula_client.podspec

@@ -6,7 +6,7 @@
 
 Pod::Spec.new do |s|
   s.name             = 'fula_client'
-  s.version          = '0.6.2'
+  s.version          = '0.6.3'
   s.summary          = 'Flutter SDK for Fula decentralized storage'
   s.description      = <<-DESC
     A Flutter plugin providing client-side encryption, metadata privacy,

packages/fula_client/pubspec.yaml

@@ -1,6 +1,6 @@
 name: fula_client
 description: Flutter SDK for Fula decentralized storage with client-side encryption, metadata privacy, and secure sharing.
-version: 0.6.2
+version: 0.6.3
 homepage: https://fx.land
 repository: https://github.com/functionland/fula-api
 issue_tracker: https://github.com/functionland/fula-api/issues