fix: do not republish files removed remotely while this client was offline

THIRD_PARTY_NOTICES.md

@@ -0,0 +1,28 @@
+# Third-party notices
+
+## node-diff3 / Project Synchrotron
+
+Portions of `src/merge-hsm/diff3.ts` are derived from node-diff3, whose diff
+implementation was extracted from Project Synchrotron.
+
+Copyright (c) 2006, 2008 Tony Garnock-Jones <tonyg@lshift.net>
+Copyright (c) 2006, 2008 LShift Ltd. <query@lshift.net>
+Copyright (c) 2019-2025 Bryan Housel <bhousel@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

__tests__/SyncStore.read-committed-paths.test.ts

@@ -0,0 +1,53 @@
+import * as Y from "yjs";
+
+import { SyncStore } from "../src/SyncStore";
+
+// The offline-removal witness (SharedFolder.captureOfflineRemovalWitness)
+// reads a scratch doc built from the persisted pre-session updates. These
+// tests pin the read against both map generations and against the
+// merge-then-read shape the witness capture actually performs.
+
+function encodeFolderState(populate: (doc: Y.Doc) => void): Uint8Array {
+	const doc = new Y.Doc();
+	populate(doc);
+	const update = Y.encodeStateAsUpdate(doc);
+	doc.destroy();
+	return update;
+}
+
+describe("SyncStore.readCommittedPaths", () => {
+	test("reads meta and legacy entries from a scratch doc", () => {
+		const update = encodeFolderState((doc) => {
+			doc.getMap("filemeta_v0").set("notes/current.md", { id: "guid-1" });
+			doc.getMap("docs").set("legacy/old.md", "guid-2");
+		});
+		const scratch = new Y.Doc();
+		Y.applyUpdate(scratch, update);
+		expect(SyncStore.readCommittedPaths(scratch)).toEqual(
+			new Set(["notes/current.md", "legacy/old.md"]),
+		);
+		scratch.destroy();
+	});
+
+	test("a path deleted in a later persisted update does not read as committed", () => {
+		const doc = new Y.Doc();
+		doc.getMap("filemeta_v0").set("moved/away.md", { id: "guid-3" });
+		const first = Y.encodeStateAsUpdate(doc);
+		doc.getMap("filemeta_v0").delete("moved/away.md");
+		const second = Y.encodeStateAsUpdate(doc, Y.encodeStateVector(new Y.Doc()));
+		doc.destroy();
+
+		const scratch = new Y.Doc();
+		Y.applyUpdate(scratch, Y.mergeUpdates([first, second]));
+		expect(SyncStore.readCommittedPaths(scratch).has("moved/away.md")).toBe(
+			false,
+		);
+		scratch.destroy();
+	});
+
+	test("empty doc reads as no committed paths", () => {
+		const scratch = new Y.Doc();
+		expect(SyncStore.readCommittedPaths(scratch).size).toBe(0);
+		scratch.destroy();
+	});
+});

jest.config.js

@@ -18,6 +18,7 @@ module.exports = {
 	testPathIgnorePatterns: ["/__tests__/mocks/", "/__tests__/merge-hsm/testing/", "archive/", ".claude"],
     globals: {
         "BUILD_TYPE": "production",
+        "GIT_TAG": "jest-test-version",
     },
 	transformIgnorePatterns: ["[\\/]node_modules[\\/](?!(yjs|lib0)[\\/])"],
 	transform: {

src/Document.ts

@@ -147,6 +147,10 @@ export class Document extends HasProvider implements IFile, HasMimeType {
 			getCurrentDiskMetadata: () =>
 				this.sharedFolder.getCurrentDiskMetadata(this),
 			isFolderConnected: () => this.sharedFolder.connected,
+			// Read the provider object at consumption time. The provider may not
+			// exist yet when the HSM is constructed and may be replaced later, so
+			// this must remain a closure over the document rather than a snapshot.
+			isProviderSynced: () => this._provider?.synced === true,
 			getPersistenceMetadata: () => ({
 				path: this.path,
 				relay: this.sharedFolder.relayId || "",
@@ -1263,11 +1267,11 @@ export class Document extends HasProvider implements IFile, HasMimeType {
 			!this._activeProviderIntegration &&
 			(this._idleProviderIntegrationRefs ?? 0) === 0
 		) {
-			this._providerIntegration.destroy();
-			this._providerIntegration = null;
 			if (disconnect) {
 				this.disconnect();
 			}
+			this._providerIntegration.destroy();
+			this._providerIntegration = null;
 		}
 	}
 

src/HasProvider.ts

@@ -16,6 +16,8 @@ import { encodeClientToken } from "./client/types";
 import type { TimeProvider } from "./TimeProvider";
 import { Awareness } from "y-protocols/awareness";
 
+declare const GIT_TAG: string;
+
 export interface Subscription {
 	on: () => void;
 	off: () => void;
@@ -53,6 +55,7 @@ function makeProvider(
 ): YSweetProvider {
 	const params = {
 		token: clientToken.token,
+		v: GIT_TAG,
 	};
 	// Configure the initial state before YSweetProvider subscribes to awareness
 	// updates. A sync-only provider then starts absent without buffering a

src/SharedFolder.ts

@@ -323,6 +323,13 @@ export class SharedFolder extends HasProvider {
 	private _resolveFirstSyncConverged: (() => void) | undefined;
 	/** Paths removed by provider-applied membership updates before convergence. */
 	private _preConvergenceRemoteDeletes: Set<string> | undefined;
+	/** Paths committed here before this session's server merge (persisted
+	 * folder doc + persisted per-document merge records). Consumed against
+	 * the converged map to catch removals this client was offline for. */
+	private _preMergeLocalPaths: Set<string> | null = null;
+	/** Witness paths absent from the converged map: excluded from scan
+	 * minting so cleanupExtraLocalFiles can trash them. */
+	private _offlineRemovedPaths: Set<string> | undefined;
 	/** Deleted paths that had a local publication hold when convergence opened. */
 	private _convergenceRemoteDeletedHolds: Map<string, string> | undefined;
 	/** One parked latch re-entry per held path. */
@@ -406,6 +413,7 @@ export class SharedFolder extends HasProvider {
 			flags().enableSyncConvergenceLatch && !authoritative;
 		if (this._syncConvergenceLatchEnabled) {
 			this._preConvergenceRemoteDeletes = new Set();
+			this._offlineRemovedPaths = new Set();
 			this._convergenceRemoteDeletedHolds = new Map();
 			this._convergenceParkedUploads = new Map();
 			this._convergencePublicationRuns = new Map();
@@ -671,6 +679,7 @@ export class SharedFolder extends HasProvider {
 				// Remote folder metadata can also land before SyncStore observers are
 				// installed, so replay both local doc discovery and file-tree sync after
 				// start() to avoid missing the first batch of remote entries.
+				this.captureOfflineRemovalWitness();
 				this.addLocalDocs();
 				await this.syncFileTree();
 				try {
@@ -1479,10 +1488,17 @@ export class SharedFolder extends HasProvider {
 	private addLocalDocs(types?: SyncType[]): void {
 		// Reconciliation is not a second source of create intent. A vault create
 		// that is still settling must be decided by its timer (or canceled by a
-		// rename/delete), rather than registered early by a scan.
+		// rename/delete), rather than registered early by a scan. A path the
+		// witness marked as removed-while-offline must not be re-minted either;
+		// its disk file is awaiting cleanup (syncStore.has covers a user
+		// re-creating the path afterward, which registers through vault events).
 		let syncTFiles = this.getSyncFiles().filter((tfile) => {
 			const vpath = this.getVirtualPath(tfile.path);
-			return !this.pendingCreates.has(vpath);
+			if (this.pendingCreates.has(vpath)) return false;
+			if (this._offlineRemovedPaths?.has(vpath) && !this.syncStore.has(vpath)) {
+				return false;
+			}
+			return true;
 		});
 		if (types) {
 			syncTFiles = syncTFiles.filter((tfile) => {
@@ -2641,9 +2657,75 @@ export class SharedFolder extends HasProvider {
 			}
 		}
 		this._preConvergenceRemoteDeletes?.clear();
+		this.routeOfflineRemovedHolds();
 		this._resolveFirstSyncConverged?.();
 	}
 
+	/**
+	 * Build the offline-removal witness: paths this client knew as committed
+	 * before this session's server merge. Sources: the folder doc's persisted
+	 * pre-session state (read from storage, so unaffected by whether the
+	 * provider merged first) and the persisted per-document merge records
+	 * (which survive a lost or reset folder database). A witness path absent
+	 * from the converged map was deleted or moved remotely while this client
+	 * was offline - republishing its disk file would resurrect it.
+	 */
+	private captureOfflineRemovalWitness(): void {
+		if (!this._syncConvergenceLatchEnabled) return;
+		const witness = new Set<string>();
+		const stored = this._persistence.initialStoredState;
+		this._persistence.initialStoredState = null;
+		if (stored) {
+			const scratch = new Y.Doc();
+			try {
+				Y.applyUpdate(scratch, stored);
+				for (const path of SyncStore.readCommittedPaths(scratch)) {
+					witness.add(path);
+				}
+			} finally {
+				scratch.destroy();
+			}
+		}
+		for (const path of this.mergeManager.getPersistedStatePaths()) {
+			witness.add(path);
+		}
+		if (witness.size === 0) return;
+
+		this._preMergeLocalPaths = witness;
+		if (this._firstSyncConverged) {
+			this.routeOfflineRemovedHolds();
+		}
+	}
+
+	/**
+	 * Consume the offline-removal witness against the converged map. Vanished
+	 * paths with a pending-upload hold are routed to the remote-deleted-hold
+	 * discard; all vanished paths are excluded from scan minting so
+	 * cleanupExtraLocalFiles can trash their disk files.
+	 */
+	private routeOfflineRemovedHolds(): void {
+		const witness = this._preMergeLocalPaths;
+		if (!witness || this.destroyed) return;
+		this._preMergeLocalPaths = null;
+
+		const removed: string[] = [];
+		for (const path of witness) {
+			if (this.syncStore.getCommittedMeta(path)) continue;
+			removed.push(path);
+			this._offlineRemovedPaths?.add(path);
+			const guid = this.pendingUpload.get(path);
+			if (guid) {
+				this._convergenceRemoteDeletedHolds?.set(path, guid);
+			}
+		}
+		if (removed.length > 0) {
+			this.warn(
+				"paths removed remotely while this client was offline; will not republish",
+				removed,
+			);
+		}
+	}
+
 	private recordPreConvergenceRemoteDeletes(delta: FolderMapDelta): void {
 		if (this._firstSyncConverged || !this._preConvergenceRemoteDeletes) return;
 		for (const entry of delta.deletes) {

src/SyncStore.ts

@@ -144,6 +144,15 @@ export class SyncStore extends Observable<SyncStore> {
 		}
 	}
 
+	/** Committed paths of a folder doc that is not this store's live doc
+	 * (e.g. a scratch doc built from persisted updates). */
+	static readCommittedPaths(doc: Y.Doc): Set<string> {
+		const paths = new Set<string>();
+		doc.getMap("docs").forEach((_guid, path) => paths.add(path));
+		doc.getMap("filemeta_v0").forEach((_meta, path) => paths.add(path));
+		return paths;
+	}
+
 	print() {
 		this.log(
 			"files",

src/client/provider.ts

@@ -19,6 +19,8 @@ import { decode as decodeCBOR } from "cbor-x";
 import { metrics, curryLog } from "../debug";
 import type { TimeProvider } from "../TimeProvider";
 
+declare const GIT_TAG: string;
+
 const providerError = curryLog("[YSweetProvider]", "error");
 const providerLog = curryLog("[YSweetProvider]", "log");
 const providerDebug = curryLog("[YSweetProvider]", "debug");
@@ -1134,7 +1136,7 @@ export class YSweetProvider extends Observable<string> {
 		while (serverUrl[serverUrl.length - 1] === "/") {
 			serverUrl = serverUrl.slice(0, serverUrl.length - 1);
 		}
-		const params = { token };
+		const params = { token, v: GIT_TAG };
 		const encodedParams = url.encodeQueryParams(params);
 		const newUrl =
 			serverUrl +