From f8092d804f635abec787b9522b10ab8eb39454ec Mon Sep 17 00:00:00 2001 From: Jeff Emmett Date: Fri, 5 Dec 2025 14:05:03 -0800 Subject: [PATCH] feat: add data safety mitigations and UI fixes for google-export branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add pre-conversion backup system for format migrations - Backups saved to `pre-conversion-backups/{roomId}/{timestamp}_{formatType}.json` - Preserves original data before any destructive conversion - Add conversion threshold guards - Abort if >10% records lost during conversion - Warn if >5% shapes lost - Full logging of before/after counts - Improve unknown format handling - Backup unknown formats instead of silently creating empty doc - Log raw document keys for investigation - Fix keyboard shortcuts dialog error - Handle tldraw i18n label objects ({default, menu}) instead of plain strings - Add getLabelString helper to safely extract string labels - Reorder Integrations tab in Settings - Google Workspace now appears first, above Obsidian and Fathom 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- src/ui/UserSettingsModal.tsx | 266 +++++++++++++++---------------- src/ui/components.tsx | 43 ++--- worker/AutomergeDurableObject.ts | 132 ++++++++++++++- 3 files changed, 284 insertions(+), 157 deletions(-) diff --git a/src/ui/UserSettingsModal.tsx b/src/ui/UserSettingsModal.tsx index fc66670..21d5a5f 100644 --- a/src/ui/UserSettingsModal.tsx +++ b/src/ui/UserSettingsModal.tsx @@ -649,8 +649,140 @@ export function UserSettingsModal({ onClose, isDarkMode, onToggleDarkMode }: Use {activeTab === 'integrations' && (
- {/* Knowledge Management Section */} + {/* Google Workspace - Primary Data Import */}

+ Google Workspace +

+ + {/* Google Workspace */} +
+
+ 🔐 +
+ Google Workspace +

+ Import Gmail, Drive, Photos & Calendar - encrypted locally +

+
+ + {googleConnected ? 'Connected' : 'Not Connected'} + +
+ + {googleConnected && totalGoogleItems > 0 && ( +
+ {googleCounts.gmail > 0 && ( + + 📧 {googleCounts.gmail} emails + + )} + {googleCounts.drive > 0 && ( + + 📁 {googleCounts.drive} files + + )} + {googleCounts.photos > 0 && ( + + 📷 {googleCounts.photos} photos + + )} + {googleCounts.calendar > 0 && ( + + 📅 {googleCounts.calendar} events + + )} +
+ )} + +

+ Your data is encrypted with AES-256 and stored only in your browser. + Choose what to share to the board. +

+ +
+ {googleConnected ? ( + <> + + + + ) : ( + + )} +
+ + {googleConnected && totalGoogleItems === 0 && ( +

+ No data imported yet. Visit /google to import. +

+ )} +
+ +
+ + {/* Knowledge Management Section */} +

Knowledge Management

@@ -845,138 +977,6 @@ export function UserSettingsModal({ onClose, isDarkMode, onToggleDarkMode }: Use )}
-
- - {/* Data Import Section */} -

- Data Import -

- - {/* Google Workspace */} -
-
- 🔐 -
- Google Workspace -

- Import Gmail, Drive, Photos & Calendar - encrypted locally -

-
- - {googleConnected ? 'Connected' : 'Not Connected'} - -
- - {googleConnected && totalGoogleItems > 0 && ( -
- {googleCounts.gmail > 0 && ( - - 📧 {googleCounts.gmail} emails - - )} - {googleCounts.drive > 0 && ( - - 📁 {googleCounts.drive} files - - )} - {googleCounts.photos > 0 && ( - - 📷 {googleCounts.photos} photos - - )} - {googleCounts.calendar > 0 && ( - - 📅 {googleCounts.calendar} events - - )} -
- )} - -

- Your data is encrypted with AES-256 and stored only in your browser. - Choose what to share to the board. -

- -
- {googleConnected ? ( - <> - - - - ) : ( - - )} -
- - {googleConnected && totalGoogleItems === 0 && ( -

- No data imported yet. Visit /google to import. -

- )} -
- {/* Future Integrations Placeholder */}

diff --git a/src/ui/components.tsx b/src/ui/components.tsx index 0fb9cd8..3c686af 100644 --- a/src/ui/components.tsx +++ b/src/ui/components.tsx @@ -406,6 +406,15 @@ function CustomSharePanel() { const actions = useActions() const [showShortcuts, setShowShortcuts] = React.useState(false) + // Helper to safely get label as string (handles i18n objects) + const getLabelString = (label: unknown, fallback: string): string => { + if (typeof label === 'string') return label + if (label && typeof label === 'object' && 'default' in label) { + return String((label as { default: string }).default) + } + return fallback + } + // Collect all tools and actions with keyboard shortcuts const allShortcuts = React.useMemo(() => { const shortcuts: { name: string; kbd: string; category: string }[] = [] @@ -416,7 +425,7 @@ function CustomSharePanel() { const tool = tools[toolId] if (tool?.kbd) { shortcuts.push({ - name: tool.label || toolId, + name: getLabelString(tool.label, toolId), kbd: tool.kbd, category: 'Tools' }) @@ -429,7 +438,7 @@ function CustomSharePanel() { const tool = tools[toolId] if (tool?.kbd) { shortcuts.push({ - name: tool.label || toolId, + name: getLabelString(tool.label, toolId), kbd: tool.kbd, category: 'Custom Tools' }) @@ -442,7 +451,7 @@ function CustomSharePanel() { const action = actions[actionId] if (action?.kbd) { shortcuts.push({ - name: action.label || actionId, + name: getLabelString(action.label, actionId), kbd: action.kbd, category: 'Actions' }) @@ -455,7 +464,7 @@ function CustomSharePanel() { const action = actions[actionId] if (action?.kbd) { shortcuts.push({ - name: action.label || actionId, + name: getLabelString(action.label, actionId), kbd: action.kbd, category: 'Custom Actions' }) @@ -662,10 +671,8 @@ export const components: TLComponents = { // MycelialIntelligence moved to permanent floating bar ].filter(tool => tool && tool.kbd) - // Get all custom actions with keyboard shortcuts + // Get only truly custom actions with keyboard shortcuts (not tldraw defaults) const customActions = [ - actions["zoom-in"], - actions["zoom-out"], actions["zoom-to-selection"], actions["copy-link-to-current-view"], actions["copy-focus-link"], @@ -676,34 +683,34 @@ export const components: TLComponents = { actions["search-shapes"], actions["llm"], actions["open-obsidian-browser"], - ].filter(action => action && action.kbd) - + ].filter(action => action && action.kbd && typeof action.label === 'string') + return ( {/* Custom Tools */} - {customTools.map(tool => ( - typeof tool.label === 'string').map(tool => ( + ))} - + {/* Custom Actions */} {customActions.map(action => ( - ))} - + {/* Default content (includes standard TLDraw shortcuts) */} diff --git a/worker/AutomergeDurableObject.ts b/worker/AutomergeDurableObject.ts index 260dc2c..68cc97a 100644 --- a/worker/AutomergeDurableObject.ts +++ b/worker/AutomergeDurableObject.ts @@ -30,6 +30,10 @@ export class AutomergeDurableObject { // Store the Automerge document ID for this room private automergeDocumentId: string | null = null + // Safety thresholds for format conversion + private static readonly CONVERSION_LOSS_THRESHOLD = 0.10 // Abort if > 10% records lost + private static readonly SHAPE_LOSS_THRESHOLD = 0.05 // Warn if > 5% shapes lost + constructor(private readonly ctx: DurableObjectState, env: Environment) { this.r2 = env.TLDRAW_BUCKET @@ -639,21 +643,35 @@ export class AutomergeDurableObject { if (Array.isArray(rawDoc)) { // This is the raw Automerge document format - convert to store format console.log(`Converting Automerge document format to store format for room ${this.roomId}`) + + // SAFETY: Create pre-conversion backup before destructive operation + await this.createPreConversionBackup(rawDoc, 'automerge-array') + + const originalShapeCount = rawDoc.filter((r: any) => r?.state?.typeName === 'shape').length initialDoc = this.convertAutomergeToStore(rawDoc) wasConverted = true - const customRecords = Object.values(initialDoc.store).filter((r: any) => + + // SAFETY: Validate conversion results + const convertedCount = Object.keys(initialDoc.store).length + const skippedCount = rawDoc.length - convertedCount + this.validateConversionResults(rawDoc.length, convertedCount, skippedCount, 'Automerge-array') + + const convertedShapeCount = Object.values(initialDoc.store).filter((r: any) => r.typeName === 'shape').length + this.validateShapeCount(originalShapeCount, convertedShapeCount, 'Automerge-array') + + const customRecords = Object.values(initialDoc.store).filter((r: any) => r.id && typeof r.id === 'string' && r.id.startsWith('obsidian_vault:') ) console.log(`Conversion completed:`, { storeKeys: Object.keys(initialDoc.store).length, - shapeCount: Object.values(initialDoc.store).filter((r: any) => r.typeName === 'shape').length, + shapeCount: convertedShapeCount, customRecordCount: customRecords.length, customRecordIds: customRecords.map((r: any) => r.id).slice(0, 5) }) } else if ((rawDoc as any).store) { // This is already in store format initialDoc = rawDoc - const customRecords = Object.values(initialDoc.store).filter((r: any) => + const customRecords = Object.values(initialDoc.store).filter((r: any) => r.id && typeof r.id === 'string' && r.id.startsWith('obsidian_vault:') ) console.log(`Document already in store format:`, { @@ -665,19 +683,42 @@ export class AutomergeDurableObject { } else if ((rawDoc as any).documents && !((rawDoc as any).store)) { // Migrate old format (documents array) to new format (store object) console.log(`Migrating old documents format to new store format for room ${this.roomId}`) + + // SAFETY: Create pre-conversion backup before destructive operation + await this.createPreConversionBackup(rawDoc, 'documents-array') + + const originalShapeCount = ((rawDoc as any).documents || []).filter((r: any) => r?.state?.typeName === 'shape').length initialDoc = this.migrateDocumentsToStore(rawDoc) wasConverted = true - const customRecords = Object.values(initialDoc.store).filter((r: any) => + + // SAFETY: Validate conversion results + const documentsArray = (rawDoc as any).documents || [] + const convertedCount = Object.keys(initialDoc.store).length + const skippedCount = documentsArray.length - convertedCount + this.validateConversionResults(documentsArray.length, convertedCount, skippedCount, 'Documents-array') + + const convertedShapeCount = Object.values(initialDoc.store).filter((r: any) => r.typeName === 'shape').length + this.validateShapeCount(originalShapeCount, convertedShapeCount, 'Documents-array') + + const customRecords = Object.values(initialDoc.store).filter((r: any) => r.id && typeof r.id === 'string' && r.id.startsWith('obsidian_vault:') ) console.log(`Migration completed:`, { storeKeys: Object.keys(initialDoc.store).length, - shapeCount: Object.values(initialDoc.store).filter((r: any) => r.typeName === 'shape').length, + shapeCount: convertedShapeCount, customRecordCount: customRecords.length, customRecordIds: customRecords.map((r: any) => r.id).slice(0, 5) }) } else { - console.log(`Unknown document format, creating new document`) + // SAFETY: Unknown format - preserve raw data and log for investigation + console.warn(`⚠️ Unknown document format for room ${this.roomId}. Preserving raw data for manual recovery.`) + console.warn(`Raw document keys: ${Object.keys(rawDoc || {}).join(', ')}`) + + // Create backup of unknown format for manual investigation + await this.createPreConversionBackup(rawDoc, 'unknown-format') + + // Create empty document but log warning + console.log(`Creating new empty document due to unknown format`) initialDoc = this.createEmptyDocument() } @@ -1047,6 +1088,85 @@ export class AutomergeDurableObject { } } + /** + * Create a pre-conversion backup of the raw document before format migration. + * This ensures we can recover data if conversion goes wrong. + */ + private async createPreConversionBackup(rawDoc: any, formatType: string): Promise { + if (!this.roomId) return false + + try { + const timestamp = new Date().toISOString().replace(/[:.]/g, '-') + const backupKey = `pre-conversion-backups/${this.roomId}/${timestamp}_${formatType}.json` + + console.log(`📦 Creating pre-conversion backup: ${backupKey}`) + + await this.r2.put(backupKey, JSON.stringify(rawDoc), { + httpMetadata: { + contentType: 'application/json' + }, + customMetadata: { + roomId: this.roomId, + formatType: formatType, + timestamp: timestamp, + reason: 'pre-conversion-safety-backup' + } + }) + + console.log(`✅ Pre-conversion backup created successfully: ${backupKey}`) + return true + } catch (error) { + console.error(`❌ Failed to create pre-conversion backup:`, error) + return false + } + } + + /** + * Validate conversion results and throw if data loss exceeds threshold. + */ + private validateConversionResults( + originalCount: number, + convertedCount: number, + skippedCount: number, + formatType: string + ): void { + if (originalCount === 0) return // Nothing to validate + + const lossRate = skippedCount / originalCount + + if (lossRate > AutomergeDurableObject.CONVERSION_LOSS_THRESHOLD) { + const errorMsg = `🚨 CONVERSION ABORTED: ${formatType} conversion would lose ${(lossRate * 100).toFixed(1)}% of records (${skippedCount}/${originalCount}). Threshold: ${(AutomergeDurableObject.CONVERSION_LOSS_THRESHOLD * 100)}%` + console.error(errorMsg) + throw new Error(errorMsg) + } + + if (lossRate > 0) { + console.warn(`⚠️ ${formatType} conversion: ${skippedCount}/${originalCount} records (${(lossRate * 100).toFixed(1)}%) could not be converted`) + } + } + + /** + * Validate shape count after conversion and warn if significant loss. + */ + private validateShapeCount( + originalShapeCount: number, + convertedShapeCount: number, + formatType: string + ): void { + if (originalShapeCount === 0) return + + const shapeLoss = originalShapeCount - convertedShapeCount + const shapeLossRate = shapeLoss / originalShapeCount + + if (shapeLossRate > AutomergeDurableObject.SHAPE_LOSS_THRESHOLD) { + console.error(`🚨 SHAPE LOSS WARNING: ${formatType} conversion lost ${shapeLoss} shapes (${(shapeLossRate * 100).toFixed(1)}%). Original: ${originalShapeCount}, After: ${convertedShapeCount}`) + } else if (shapeLoss > 0) { + console.warn(`⚠️ ${formatType} conversion: ${shapeLoss} shapes could not be converted`) + } else { + console.log(`✅ ${formatType} conversion: All ${originalShapeCount} shapes preserved`) + } + } + private async updateDocument(newDoc: any) { // CRITICAL: Wait for R2 load to complete before processing updates // This ensures we have all shapes from R2 before merging client updates