From 122a2a16823efd35a569c76bce27b52c69809413 Mon Sep 17 00:00:00 2001 From: Jeff Emmett Date: Wed, 3 Sep 2025 16:12:28 +0200 Subject: [PATCH] transcription with webspeechAPI almost there (sync problem) --- docs/TRANSCRIPTION_TOOL.md | 171 ++++++++ package.json | 2 +- src/components/TranscribeComponent.tsx | 518 +++++++++++++++++++++++++ src/css/transcribe.css | 261 +++++++++++++ src/routes/Board.tsx | 5 + src/routes/Presentations.tsx | 45 +-- src/shapes/TranscribeShapeUtil.tsx | 47 +++ src/shapes/VideoChatShapeUtil.tsx | 446 +++++++++++---------- src/tools/TranscribeTool.ts | 7 + src/types/webspeech.d.ts | 56 +++ worker/TldrawDurableObject.ts | 5 + worker/shapes/TranscribeShapeUtil.ts | 41 ++ 12 files changed, 1357 insertions(+), 247 deletions(-) create mode 100644 docs/TRANSCRIPTION_TOOL.md create mode 100644 src/components/TranscribeComponent.tsx create mode 100644 src/css/transcribe.css create mode 100644 src/shapes/TranscribeShapeUtil.tsx create mode 100644 src/tools/TranscribeTool.ts create mode 100644 src/types/webspeech.d.ts create mode 100644 worker/shapes/TranscribeShapeUtil.ts diff --git a/docs/TRANSCRIPTION_TOOL.md b/docs/TRANSCRIPTION_TOOL.md new file mode 100644 index 0000000..6e87367 --- /dev/null +++ b/docs/TRANSCRIPTION_TOOL.md @@ -0,0 +1,171 @@ +# Transcription Tool for Canvas + +The Transcription Tool is a powerful feature that allows you to transcribe audio from participants in your Canvas sessions using the Web Speech API. This tool provides real-time speech-to-text conversion, making it easy to capture and document conversations, presentations, and discussions. + +## Features + +### 🎀 Real-time Transcription +- Live speech-to-text conversion using the Web Speech API +- Support for multiple languages including English, Spanish, French, German, and more +- Continuous recording with interim and final results + +### 🌐 Multi-language Support +- **English (US/UK)**: Primary language support +- **European Languages**: Spanish, French, German, Italian, Portuguese +- **Asian Languages**: Japanese, Korean, Chinese (Simplified) +- Easy language switching during recording sessions + +### πŸ‘₯ Participant Management +- Automatic participant detection and tracking +- Individual transcript tracking for each speaker +- Visual indicators for speaking status + +### πŸ“ Transcript Management +- Real-time transcript display with auto-scroll +- Clear transcript functionality +- Download transcripts as text files +- Persistent storage within the Canvas session + +### βš™οΈ Advanced Controls +- Auto-scroll toggle for better reading experience +- Recording start/stop controls +- Error handling and status indicators +- Microphone permission management + +## How to Use + +### 1. Adding the Tool to Your Canvas + +1. In your Canvas session, look for the **Transcribe** tool in the toolbar +2. Click on the Transcribe tool icon +3. Click and drag on the canvas to create a transcription widget +4. The widget will appear with default dimensions (400x300 pixels) + +### 2. Starting a Recording Session + +1. **Select Language**: Choose your preferred language from the dropdown menu +2. **Enable Auto-scroll**: Check the auto-scroll checkbox for automatic scrolling +3. **Start Recording**: Click the "🎀 Start Recording" button +4. **Grant Permissions**: Allow microphone access when prompted by your browser + +### 3. During Recording + +- **Live Transcription**: See real-time text as people speak +- **Participant Tracking**: Monitor who is speaking +- **Status Indicators**: Red dot shows active recording +- **Auto-scroll**: Transcript automatically scrolls to show latest content + +### 4. Managing Your Transcript + +- **Stop Recording**: Click "⏹️ Stop Recording" to end the session +- **Clear Transcript**: Use "πŸ—‘οΈ Clear" to reset the transcript +- **Download**: Click "πŸ’Ύ Download" to save as a text file + +## Browser Compatibility + +### βœ… Supported Browsers +- **Chrome/Chromium**: Full support with `webkitSpeechRecognition` +- **Edge (Chromium)**: Full support +- **Safari**: Limited support (may require additional setup) + +### ❌ Unsupported Browsers +- **Firefox**: No native support for Web Speech API +- **Internet Explorer**: No support + +### πŸ”§ Recommended Setup +For the best experience, use **Chrome** or **Chromium-based browsers** with: +- Microphone access enabled +- HTTPS connection (required for microphone access) +- Stable internet connection + +## Technical Details + +### Web Speech API Integration +The tool uses the Web Speech API's `SpeechRecognition` interface: +- **Continuous Mode**: Enables ongoing transcription +- **Interim Results**: Shows partial results in real-time +- **Language Detection**: Automatically adjusts to selected language +- **Error Handling**: Graceful fallback for unsupported features + +### Audio Processing +- **Microphone Access**: Secure microphone permission handling +- **Audio Stream Management**: Proper cleanup of audio resources +- **Quality Optimization**: Optimized for voice recognition + +### Data Persistence +- **Session Storage**: Transcripts persist during the Canvas session +- **Shape Properties**: All settings and data stored in the Canvas shape +- **Real-time Updates**: Changes sync across all participants + +## Troubleshooting + +### Common Issues + +#### "Speech recognition not supported in this browser" +- **Solution**: Use Chrome or a Chromium-based browser +- **Alternative**: Check if you're using the latest browser version + +#### "Unable to access microphone" +- **Solution**: Check browser permissions for microphone access +- **Alternative**: Ensure you're on an HTTPS connection + +#### Poor transcription quality +- **Solutions**: + - Speak clearly and at a moderate pace + - Reduce background noise + - Ensure good microphone positioning + - Check internet connection stability + +#### Language not working correctly +- **Solution**: Verify the selected language matches the spoken language +- **Alternative**: Try restarting the recording session + +### Performance Tips + +1. **Close unnecessary tabs** to free up system resources +2. **Use a good quality microphone** for better accuracy +3. **Minimize background noise** in your environment +4. **Speak at a natural pace** - not too fast or slow +5. **Ensure stable internet connection** for optimal performance + +## Future Enhancements + +### Planned Features +- **Speaker Identification**: Advanced voice recognition for multiple speakers +- **Export Formats**: Support for PDF, Word, and other document formats +- **Real-time Translation**: Multi-language translation capabilities +- **Voice Commands**: Canvas control through voice commands +- **Cloud Storage**: Automatic transcript backup and sharing + +### Integration Possibilities +- **Daily.co Integration**: Enhanced participant detection from video sessions +- **AI Enhancement**: Improved accuracy using machine learning +- **Collaborative Editing**: Real-time transcript editing by multiple users +- **Search and Indexing**: Full-text search within transcripts + +## Support and Feedback + +If you encounter issues or have suggestions for improvements: + +1. **Check Browser Compatibility**: Ensure you're using a supported browser +2. **Review Permissions**: Verify microphone access is granted +3. **Check Network**: Ensure stable internet connection +4. **Report Issues**: Contact the development team with detailed error information + +## Privacy and Security + +### Data Handling +- **Local Processing**: Speech recognition happens locally in your browser +- **No Cloud Storage**: Transcripts are not automatically uploaded to external services +- **Session Privacy**: Data is only shared within your Canvas session +- **User Control**: You control when and what to record + +### Best Practices +- **Inform Participants**: Let others know when recording +- **Respect Privacy**: Don't record sensitive or confidential information +- **Secure Sharing**: Be careful when sharing transcript files +- **Regular Cleanup**: Clear transcripts when no longer needed + +--- + +*The Transcription Tool is designed to enhance collaboration and documentation in Canvas sessions. Use it responsibly and respect the privacy of all participants.* diff --git a/package.json b/package.json index 39ce647..28cab65 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "dev": "concurrently --kill-others --names client,worker --prefix-colors blue,red \"npm run dev:client\" \"npm run dev:worker\"", "dev:client": "vite --host --port 5173", - "dev:worker": "wrangler dev --remote --port 5172 --ip 0.0.0.0", + "dev:worker": "wrangler dev --local --port 5172 --ip 0.0.0.0", "build": "tsc && vite build", "preview": "vite preview", "deploy": "tsc && vite build && vercel deploy --prod && wrangler deploy", diff --git a/src/components/TranscribeComponent.tsx b/src/components/TranscribeComponent.tsx new file mode 100644 index 0000000..62825dc --- /dev/null +++ b/src/components/TranscribeComponent.tsx @@ -0,0 +1,518 @@ +import React, { useEffect, useRef, useState, useCallback } from 'react' +import { ITranscribeShape, TranscribeShapeUtil } from '../shapes/TranscribeShapeUtil' +import { useEditor } from '@tldraw/tldraw' + +interface TranscribeComponentProps { + shape: ITranscribeShape + util: TranscribeShapeUtil +} + +interface Participant { + id: string + name: string + isSpeaking: boolean + lastSpoken: string + transcript: string +} + +export function TranscribeComponent({ shape }: TranscribeComponentProps) { + const editor = useEditor() + const [isRecording, setIsRecording] = useState(shape.props.isRecording) + const [transcript, setTranscript] = useState(shape.props.transcript) + const [participants, setParticipants] = useState(() => + shape.props.participants.map(p => ({ + id: p.id, + name: p.name, + isSpeaking: p.isSpeaking, + lastSpoken: p.lastSpoken, + transcript: '' + })) + ) + const [isPaused, setIsPaused] = useState(false) + const [userHasScrolled, setUserHasScrolled] = useState(false) + const [error, setError] = useState(null) + const [isSupported, setIsSupported] = useState(false) + + const transcriptRef = useRef(null) + const recognitionRef = useRef(null) + const mediaStreamRef = useRef(null) + const localTranscriptRef = useRef('') + + // Immediate update for critical state changes (recording start/stop) + const updateShapePropsImmediate = useCallback((updates: Partial) => { + try { + // Only update if the editor is still valid and the shape exists + const currentShape = editor.getShape(shape.id) + if (currentShape) { + console.log('πŸ”„ Updating shape props immediately:', updates) + editor.updateShape({ + id: shape.id, + type: shape.type, + props: { + ...shape.props, + ...updates + } + }) + console.log('βœ… Shape props updated successfully') + } else { + console.log('⚠️ Shape no longer exists, skipping immediate update') + } + } catch (error) { + console.error('❌ Error in immediate update:', error) + console.error('❌ Update data:', updates) + console.error('❌ Shape data:', shape) + } + }, [editor, shape]) + + // Simple transcript update strategy like other shapes use + const updateTranscriptLocal = useCallback((newTranscript: string) => { + console.log('πŸ“ Updating transcript:', newTranscript.length, 'chars') + + // Always update local state immediately for responsive UI + localTranscriptRef.current = newTranscript + + // Use requestAnimationFrame for smooth updates like PromptShape does + requestAnimationFrame(() => { + try { + const currentShape = editor.getShape(shape.id) + if (currentShape) { + console.log('πŸ”„ Updating transcript in shape:', { + transcriptLength: newTranscript.length, + participantsCount: participants.length, + shapeId: shape.id + }) + editor.updateShape({ + id: shape.id, + type: shape.type, + props: { + ...shape.props, + transcript: newTranscript, + participants: participants + } + }) + console.log('βœ… Transcript updated successfully') + } else { + console.log('⚠️ Shape not found for transcript update') + } + } catch (error) { + console.error('❌ Error updating transcript:', error) + console.error('❌ Transcript data:', newTranscript.slice(0, 100) + '...') + console.error('❌ Participants data:', participants) + } + }) + }, [editor, shape, participants]) + + // Check if Web Speech API is supported + useEffect(() => { + const checkSupport = () => { + if ('webkitSpeechRecognition' in window) { + setIsSupported(true) + return (window as any).webkitSpeechRecognition + } else if ('SpeechRecognition' in window) { + setIsSupported(true) + return (window as any).SpeechRecognition + } else { + setIsSupported(false) + setError('Speech recognition not supported in this browser') + return null + } + } + + const SpeechRecognition = checkSupport() + if (SpeechRecognition) { + recognitionRef.current = new SpeechRecognition() + setupSpeechRecognition() + } + }, []) + + + + const setupSpeechRecognition = useCallback(() => { + console.log('πŸ”§ Setting up speech recognition...') + if (!recognitionRef.current) { + console.log('❌ No recognition ref available') + return + } + + const recognition = recognitionRef.current + console.log('βœ… Recognition ref found, configuring...') + + recognition.continuous = true + recognition.interimResults = true + recognition.lang = 'en-US' // Fixed to English + + console.log('πŸ”§ Recognition configured:', { + continuous: recognition.continuous, + interimResults: recognition.interimResults, + lang: recognition.lang + }) + + recognition.onstart = () => { + console.log('🎯 Speech recognition onstart event fired') + console.log('Setting isRecording to true') + setIsRecording(true) + updateShapePropsImmediate({ isRecording: true }) + console.log('βœ… Recording state updated') + } + + recognition.onresult = (event: any) => { + console.log('🎀 Speech recognition onresult event fired', event) + console.log('Event details:', { + resultIndex: event.resultIndex, + resultsLength: event.results.length, + hasResults: !!event.results + }) + + let finalTranscript = '' + let interimTranscript = '' + + for (let i = event.resultIndex; i < event.results.length; i++) { + const transcript = event.results[i][0].transcript + console.log(`πŸ“ Result ${i}: "${transcript}" (final: ${event.results[i].isFinal})`) + if (event.results[i].isFinal) { + finalTranscript += transcript + } else { + interimTranscript += transcript + } + } + + if (finalTranscript) { + console.log('βœ… Final transcript:', finalTranscript) + // Use functional update to avoid dependency on current transcript state + setTranscript(prevTranscript => { + const newTranscript = prevTranscript + finalTranscript + '\n' + console.log('πŸ“ Updating transcript:', { + prevLength: prevTranscript.length, + newLength: newTranscript.length, + prevText: prevTranscript.slice(-50), // Last 50 chars + newText: newTranscript.slice(-50) // Last 50 chars + }) + // Update shape props with the new transcript using local-first update + updateTranscriptLocal(newTranscript) + return newTranscript + }) + + // Add to participants if we can identify who's speaking + addParticipantTranscript('Speaker', finalTranscript) + } + + if (interimTranscript) { + console.log('⏳ Interim transcript:', interimTranscript) + } + + // Smart auto-scroll: only scroll if user hasn't manually scrolled away + if (!userHasScrolled && transcriptRef.current) { + transcriptRef.current.scrollTop = transcriptRef.current.scrollHeight + console.log('πŸ“œ Auto-scrolled to bottom') + } + } + + recognition.onerror = (event: any) => { + console.error('❌ Speech recognition error:', event.error) + setError(`Recognition error: ${event.error}`) + setIsRecording(false) + updateShapePropsImmediate({ isRecording: false }) + } + + recognition.onend = () => { + console.log('πŸ›‘ Speech recognition ended') + setIsRecording(false) + updateShapePropsImmediate({ isRecording: false }) + } + }, [updateShapePropsImmediate]) + + const startRecording = useCallback(async () => { + try { + console.log('🎀 Starting recording...') + console.log('Recognition ref exists:', !!recognitionRef.current) + console.log('Current recognition state:', recognitionRef.current?.state || 'unknown') + + // Request microphone permission + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }) + mediaStreamRef.current = stream + console.log('βœ… Microphone access granted') + + if (recognitionRef.current) { + console.log('🎯 Starting speech recognition...') + console.log('Recognition settings:', { + continuous: recognitionRef.current.continuous, + interimResults: recognitionRef.current.interimResults, + lang: recognitionRef.current.lang + }) + recognitionRef.current.start() + console.log('βœ… Speech recognition start() called') + } else { + console.error('❌ Recognition ref is null') + setError('Speech recognition not initialized') + } + } catch (err) { + console.error('❌ Error accessing microphone:', err) + setError('Unable to access microphone. Please check permissions.') + } + }, []) + + const pauseRecording = useCallback(() => { + if (recognitionRef.current && isRecording) { + console.log('⏸️ Pausing transcription...') + recognitionRef.current.stop() + setIsPaused(true) + } + }, [isRecording]) + + const resumeRecording = useCallback(async () => { + if (recognitionRef.current && isPaused) { + console.log('▢️ Resuming transcription...') + try { + recognitionRef.current.start() + setIsPaused(false) + } catch (err) { + console.error('❌ Error resuming transcription:', err) + setError('Unable to resume transcription') + } + } + }, [isPaused]) + + // Auto-start transcription if isRecording is true from the beginning + useEffect(() => { + console.log('πŸ” Auto-start useEffect triggered:', { + isSupported, + hasRecognition: !!recognitionRef.current, + shapeIsRecording: shape.props.isRecording, + componentIsRecording: isRecording + }) + + if (isSupported && recognitionRef.current && shape.props.isRecording && !isRecording) { + console.log('πŸš€ Auto-starting transcription from shape props...') + setTimeout(() => { + startRecording() + }, 1000) // Small delay to ensure everything is set up + } + }, [isSupported, startRecording, shape.props.isRecording, isRecording]) + + // Add global error handler for sync errors + useEffect(() => { + const handleGlobalError = (event: ErrorEvent) => { + if (event.message && event.message.includes('INVALID_RECORD')) { + console.error('🚨 INVALID_RECORD sync error detected:', event.message) + console.error('🚨 Error details:', event.error) + setError('Sync error detected. Please refresh the page.') + } + } + + window.addEventListener('error', handleGlobalError) + return () => window.removeEventListener('error', handleGlobalError) + }, []) + + const addParticipantTranscript = useCallback((speakerName: string, text: string) => { + setParticipants(prev => { + const existing = prev.find(p => p.name === speakerName) + const newParticipants = existing + ? prev.map(p => + p.name === speakerName + ? { ...p, lastSpoken: text, transcript: p.transcript + '\n' + text } + : p + ) + : [...prev, { + id: Date.now().toString(), + name: speakerName, + isSpeaking: false, + lastSpoken: text, + transcript: text + }] + + // Don't update shape props for participants immediately - let it batch with transcript + // This reduces the number of shape updates + + return newParticipants + }) + }, []) + + const clearTranscript = useCallback(() => { + setTranscript('') + setParticipants([]) + editor.updateShape({ + id: shape.id, + type: shape.type, + props: { + ...shape.props, + transcript: '', + participants: [] + } + }) + }, [editor, shape]) + + const copyTranscript = useCallback(async () => { + try { + await navigator.clipboard.writeText(transcript) + console.log('βœ… Transcript copied to clipboard') + // You could add a temporary "Copied!" message here if desired + } catch (err) { + console.error('❌ Failed to copy transcript:', err) + // Fallback for older browsers + const textArea = document.createElement('textarea') + textArea.value = transcript + document.body.appendChild(textArea) + textArea.select() + document.execCommand('copy') + document.body.removeChild(textArea) + console.log('βœ… Transcript copied using fallback method') + } + }, [transcript]) + + // Cleanup on unmount + useEffect(() => { + return () => { + if (recognitionRef.current) { + recognitionRef.current.stop() + } + if (mediaStreamRef.current) { + mediaStreamRef.current.getTracks().forEach(track => track.stop()) + } + // Cleanup completed + // Ensure final transcript is saved + if (localTranscriptRef.current) { + editor.updateShape({ + id: shape.id, + type: shape.type, + props: { + ...shape.props, + transcript: localTranscriptRef.current + } + }) + } + } + }, [editor, shape]) + + // Handle scroll events to detect user scrolling + const handleScroll = useCallback(() => { + if (transcriptRef.current) { + const { scrollTop, scrollHeight, clientHeight } = transcriptRef.current + const isAtBottom = scrollTop + clientHeight >= scrollHeight - 10 // 10px threshold + + if (isAtBottom) { + setUserHasScrolled(false) // User is back at bottom, re-enable auto-scroll + } else { + setUserHasScrolled(true) // User has scrolled away, disable auto-scroll + } + } + }, []) + + if (!isSupported) { + return ( +
+
+

Speech recognition not supported in this browser.

+

Please use Chrome or a WebKit-based browser.

+
+
+ ) + } + + return ( +
+ {/* Header */} +
+

Live Transcription

+
+ + {/* Recording Controls - Simplified */} +
+ {!isRecording && !isPaused ? ( + + ) : isPaused ? ( + + ) : ( + + )} + + +
+ + {/* Status */} +
+ {isRecording && !isPaused && ( +
+ πŸ”΄ Recording... +
+ )} + {isPaused && ( +
+ ⏸️ Paused +
+ )} + {error && ( +
+ ⚠️ {error} +
+ )} +
+ + {/* Participants */} + {participants.length > 0 && ( +
+

Participants ({participants.length})

+
+ {participants.map(participant => ( +
+ {participant.name} + + {participant.isSpeaking ? 'πŸ”Š Speaking' : 'πŸ”‡ Silent'} + +
+ ))} +
+
+ )} + + {/* Transcript */} +
+

Live Transcript

+
+ {(transcript || localTranscriptRef.current) ? ( + 
+              {transcript || localTranscriptRef.current}
+              {/* Debug info */}
+              

+                Debug: {transcript.length} chars (local: {localTranscriptRef.current.length}), 
+                isRecording: {isRecording.toString()}
+              

+
+ ) : ( +

+ Start recording to see live transcription... (Debug: transcript length = {transcript.length}) +

+ )} +
+
+
+ ) +} diff --git a/src/css/transcribe.css b/src/css/transcribe.css new file mode 100644 index 0000000..67deab0 --- /dev/null +++ b/src/css/transcribe.css @@ -0,0 +1,261 @@ +/* Transcription Component Styles */ +.transcribe-container { + background: #ffffff; + border: 2px solid #e2e8f0; + border-radius: 12px; + padding: 16px; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1); + overflow: hidden; + display: flex; + flex-direction: column; + gap: 12px; +} + +.transcribe-header { + display: flex; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid #e2e8f0; + padding-bottom: 12px; + margin-bottom: 12px; +} + +.transcribe-header h3 { + margin: 0; + color: #1f2937; + font-size: 18px; + font-weight: 600; +} + +.transcribe-controls { + display: flex; + gap: 8px; + align-items: center; + flex-wrap: wrap; +} + +.transcribe-controls select { + padding: 6px 8px; + border: 1px solid #d1d5db; + border-radius: 6px; + background: #ffffff; + font-size: 14px; + color: #374151; +} + +.transcribe-controls label { + display: flex; + align-items: center; + gap: 4px; + font-size: 14px; + color: #6b7280; + cursor: pointer; +} + +.transcribe-controls input[type="checkbox"] { + margin: 0; + cursor: pointer; +} + +.transcribe-btn { + padding: 8px 16px; + border: none; + border-radius: 6px; + font-size: 14px; + font-weight: 500; + cursor: pointer; + transition: all 0.2s ease; + display: flex; + align-items: center; + gap: 6px; +} + +.transcribe-btn:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.start-btn { + background: #10b981; + color: white; +} + +.start-btn:hover:not(:disabled) { + background: #059669; + transform: translateY(-1px); +} + +.stop-btn { + background: #ef4444; + color: white; +} + +.stop-btn:hover:not(:disabled) { + background: #dc2626; + transform: translateY(-1px); +} + +.clear-btn { + background: #6b7280; + color: white; +} + +.clear-btn:hover:not(:disabled) { + background: #4b5563; + transform: translateY(-1px); +} + +.download-btn { + background: #3b82f6; + color: white; +} + +.download-btn:hover:not(:disabled) { + background: #2563eb; + transform: translateY(-1px); +} + +.transcribe-status { + display: flex; + gap: 12px; + align-items: center; + font-size: 14px; +} + +.recording-indicator { + display: flex; + align-items: center; + gap: 6px; + color: #dc2626; + font-weight: 500; +} + +.pulse { + animation: pulse 2s infinite; +} + +@keyframes pulse { + 0% { opacity: 1; } + 50% { opacity: 0.5; } + 100% { opacity: 1; } +} + +.error-message { + color: #dc2626; + background: #fef2f2; + padding: 8px 12px; + border-radius: 6px; + border: 1px solid #fecaca; +} + +.participants-section { + border-top: 1px solid #e2e8f0; + padding-top: 12px; +} + +.participants-section h4 { + margin: 0 0 8px 0; + color: #374151; + font-size: 16px; + font-weight: 600; +} + +.participants-list { + display: flex; + flex-direction: column; + gap: 6px; +} + +.participant { + display: flex; + justify-content: space-between; + align-items: center; + padding: 6px 8px; + background: #f9fafb; + border-radius: 4px; + font-size: 14px; +} + +.participant-name { + font-weight: 500; + color: #374151; +} + +.participant-status { + font-size: 12px; + color: #6b7280; +} + +.transcript-section { + flex: 1; + display: flex; + flex-direction: column; + min-height: 0; +} + +.transcript-section h4 { + margin: 0 0 8px 0; + color: #374151; + font-size: 16px; + font-weight: 600; +} + +.transcript-content { + background: #f9fafb; + border: 1px solid #e2e8f0; + border-radius: 6px; + padding: 12px; + overflow-y: auto; + flex: 1; +} + +.transcript-text { + margin: 0; + white-space: pre-wrap; + font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace; + font-size: 14px; + line-height: 1.5; + color: #374151; +} + +.transcript-placeholder { + margin: 0; + color: #9ca3af; + font-style: italic; + text-align: center; + padding: 20px; +} + +.transcribe-error { + text-align: center; + padding: 20px; + color: #6b7280; +} + +.transcribe-error p { + margin: 8px 0; +} + +/* Responsive adjustments */ +@media (max-width: 480px) { + .transcribe-container { + padding: 12px; + gap: 8px; + } + + .transcribe-header { + flex-direction: column; + align-items: flex-start; + gap: 8px; + } + + .transcribe-controls { + justify-content: flex-start; + } + + .transcribe-btn { + padding: 6px 12px; + font-size: 13px; + } +} diff --git a/src/routes/Board.tsx b/src/routes/Board.tsx index 3cf0992..241a0a2 100644 --- a/src/routes/Board.tsx +++ b/src/routes/Board.tsx @@ -6,6 +6,8 @@ import { ChatBoxTool } from "@/tools/ChatBoxTool" import { ChatBoxShape } from "@/shapes/ChatBoxShapeUtil" import { VideoChatTool } from "@/tools/VideoChatTool" import { VideoChatShape } from "@/shapes/VideoChatShapeUtil" +import { TranscribeTool } from "@/tools/TranscribeTool" +import { TranscribeShapeUtil } from "@/shapes/TranscribeShapeUtil" import { multiplayerAssetStore } from "../utils/multiplayerAssetStore" import { EmbedShape } from "@/shapes/EmbedShapeUtil" import { EmbedTool } from "@/tools/EmbedTool" @@ -46,6 +48,7 @@ import { CmdK } from "@/CmdK" import "react-cmdk/dist/cmdk.css" import "@/css/style.css" +import "@/css/transcribe.css" const collections: Collection[] = [GraphLayoutCollection] import { useAuth } from "../context/AuthContext" @@ -64,6 +67,7 @@ const customShapeUtils = [ MarkdownShape, PromptShape, SharedPianoShape, + TranscribeShapeUtil, ] const customTools = [ ChatBoxTool, @@ -75,6 +79,7 @@ const customTools = [ PromptShapeTool, SharedPianoTool, GestureTool, + TranscribeTool, ] export function Board() { diff --git a/src/routes/Presentations.tsx b/src/routes/Presentations.tsx index d331974..2b6433d 100644 --- a/src/routes/Presentations.tsx +++ b/src/routes/Presentations.tsx @@ -18,6 +18,28 @@ export function Presentations() {
+
+

Psilocybernetics: The Emergence of Institutional Neuroplasticity

+

Exploring the intersection of mycelium and cybernetic institutional design

+
+
+ - - {/* Recording Button */} - {shape.props.enableRecording && ( - - )} - - {/* Test Button - Always visible for debugging */} + {/* Transcription Button - Above video */} - {/* Transcription Button - Only for owners */} - {(() => { - console.log('πŸ” Checking transcription button conditions:'); - console.log('enableTranscription:', shape.props.enableTranscription); - console.log('isOwner:', shape.props.isOwner); - console.log('Button should render:', shape.props.enableTranscription && shape.props.isOwner); - return shape.props.enableTranscription && shape.props.isOwner; - })() && ( + {/* Video Container */} +
+ + + {/* Test Button - Always visible for debugging */} - )} - {/* Transcription History */} - {shape.props.transcriptionHistory.length > 0 && ( + {/* Transcription History */} + {shape.props.transcriptionHistory.length > 0 && ( +
+
+ Live Transcription: +
+ {shape.props.transcriptionHistory.slice(-10).map((msg) => ( +
+ + {msg.sender}: + {" "} + {msg.message} +
+ ))} +
+ )} +
+ + {/* URL Link - Below video */} +
+

{ + e.preventDefault(); + e.stopPropagation(); + if (roomUrl) { + try { + await navigator.clipboard.writeText(roomUrl); + console.log('βœ… Link copied to clipboard:', roomUrl); + + // Show temporary "link copied" message + const messageEl = document.getElementById(`copy-message-${shape.id}`); + if (messageEl) { + messageEl.style.opacity = "1"; + setTimeout(() => { + messageEl.style.opacity = "0"; + }, 2000); + } + } catch (err) { + console.error('❌ Failed to copy link:', err); + // Fallback for older browsers + const textArea = document.createElement('textarea'); + textArea.value = roomUrl; + document.body.appendChild(textArea); + textArea.select(); + document.execCommand('copy'); + document.body.removeChild(textArea); + } + } + }} + style={{ + margin: "8px 0 0 0", + padding: "4px 8px", + background: "rgba(255, 255, 255, 0.9)", + borderRadius: "4px", + fontSize: "12px", + pointerEvents: "all", + cursor: "pointer", + userSelect: "none", + border: "1px solid #e0e0e0", + transition: "background-color 0.2s ease", + }} + onMouseEnter={(e) => { + e.currentTarget.style.backgroundColor = "rgba(240, 240, 240, 0.9)"; + }} + onMouseLeave={(e) => { + e.currentTarget.style.backgroundColor = "rgba(255, 255, 255, 0.9)"; + }} + > + url: {roomUrl} + {shape.props.isOwner && " (Owner)"} +

+ + {/* "Link Copied" message */}
-
- Live Transcription: -
- {shape.props.transcriptionHistory.slice(-10).map((msg) => ( -
- - {msg.sender}: - {" "} - {msg.message} -
- ))} + Link Copied!
- )} - -

- url: {roomUrl} - {shape.props.isOwner && " (Owner)"} -

+
) } diff --git a/src/tools/TranscribeTool.ts b/src/tools/TranscribeTool.ts new file mode 100644 index 0000000..5d85e76 --- /dev/null +++ b/src/tools/TranscribeTool.ts @@ -0,0 +1,7 @@ +import { BaseBoxShapeTool } from "tldraw" + +export class TranscribeTool extends BaseBoxShapeTool { + static override id = "Transcribe" + shapeType = "Transcribe" + override initial = "idle" +} diff --git a/src/types/webspeech.d.ts b/src/types/webspeech.d.ts new file mode 100644 index 0000000..2d6d790 --- /dev/null +++ b/src/types/webspeech.d.ts @@ -0,0 +1,56 @@ +// Web Speech API TypeScript declarations +interface SpeechRecognition extends EventTarget { + continuous: boolean + interimResults: boolean + lang: string + start(): void + stop(): void + abort(): void + onstart: ((this: SpeechRecognition, ev: Event) => any) | null + onresult: ((this: SpeechRecognition, ev: SpeechRecognitionEvent) => any) | null + onerror: ((this: SpeechRecognition, ev: SpeechRecognitionErrorEvent) => any) | null + onend: ((this: SpeechRecognition, ev: Event) => any) | null +} + +interface SpeechRecognitionEvent extends Event { + resultIndex: number + results: SpeechRecognitionResultList +} + +interface SpeechRecognitionErrorEvent extends Event { + error: string + message: string +} + +interface SpeechRecognitionResultList { + length: number + item(index: number): SpeechRecognitionResult + [index: number]: SpeechRecognitionResult +} + +interface SpeechRecognitionResult { + length: number + item(index: number): SpeechRecognitionAlternative + [index: number]: SpeechRecognitionAlternative + isFinal: boolean +} + +interface SpeechRecognitionAlternative { + transcript: string + confidence: number +} + +declare var SpeechRecognition: { + prototype: SpeechRecognition + new(): SpeechRecognition +} + +declare var webkitSpeechRecognition: { + prototype: SpeechRecognition + new(): SpeechRecognition +} + +interface Window { + SpeechRecognition: typeof SpeechRecognition + webkitSpeechRecognition: typeof webkitSpeechRecognition +} diff --git a/worker/TldrawDurableObject.ts b/worker/TldrawDurableObject.ts index 74073d6..c69c1b9 100644 --- a/worker/TldrawDurableObject.ts +++ b/worker/TldrawDurableObject.ts @@ -12,6 +12,7 @@ import { MycrozineTemplateShape } from "./shapes/MycrozineTemplateShapeUtil" import { SlideShape } from "./shapes/SlideShapeUtil" import { PromptShape } from "./shapes/PromptShapeUtil" import { SharedPianoShape } from "./shapes/SharedPianoShapeUtil" +import { TranscribeShape } from "./shapes/TranscribeShapeUtil" // Lazy load TLDraw dependencies to avoid startup timeouts let customSchema: any = null @@ -56,6 +57,10 @@ async function getTldrawDependencies() { props: SharedPianoShape.props, migrations: SharedPianoShape.migrations, }, + Transcribe: { + props: TranscribeShape.props, + migrations: TranscribeShape.migrations, + }, }, bindings: defaultBindingSchemas, }) diff --git a/worker/shapes/TranscribeShapeUtil.ts b/worker/shapes/TranscribeShapeUtil.ts new file mode 100644 index 0000000..83f679b --- /dev/null +++ b/worker/shapes/TranscribeShapeUtil.ts @@ -0,0 +1,41 @@ +import { BaseBoxShapeUtil, TLBaseShape } from "tldraw" + +export type ITranscribeShape = TLBaseShape< + "Transcribe", + { + w: number + h: number + isRecording: boolean + transcript: string + participants: Array<{ + id: string + name: string + isSpeaking: boolean + lastSpoken: string + }> + language: string + } +> + +export class TranscribeShape extends BaseBoxShapeUtil { + static override type = "Transcribe" + + override getDefaultProps(): ITranscribeShape["props"] { + return { + w: 400, + h: 300, + isRecording: false, + transcript: "", + participants: [], + language: "en-US", + } + } + + override indicator(_shape: ITranscribeShape) { + return null // Simplified for worker + } + + override component(_shape: ITranscribeShape) { + return null // No React components in worker + } +}