From 8f22b8baa751554a8e07b9a7159a777ab57b53c1 Mon Sep 17 00:00:00 2001
From: Jeff Emmett <jeffemmett@gmail.com>
Date: Wed, 10 Dec 2025 10:27:44 -0800
Subject: [PATCH] feat: improve mobile touch/pen interactions across custom
 tools
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Add onTouchStart/onTouchEnd handlers to all interactive elements
- Add touchAction: 'manipulation' CSS to prevent 300ms click delay
- Increase minimum touch target sizes to 44px for accessibility
- Fix ImageGen: Generate button, Copy/Download/Delete, input field
- Fix VideoGen: Upload, URL input, prompt, duration, Generate button
- Fix Transcription: Start/Stop/Pause buttons, textarea, Save/Cancel
- Fix Multmux: Create Session, Refresh, session list, input fields

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
---
 src/shapes/ImageGenShapeUtil.tsx      | 36 +++++++++++++++
 src/shapes/MultmuxShapeUtil.tsx       | 38 ++++++++++++++++
 src/shapes/TranscriptionShapeUtil.tsx | 51 ++++++++++++++++++---
 src/shapes/VideoGenShapeUtil.tsx      | 65 ++++++++++++++++++++++-----
 4 files changed, 173 insertions(+), 17 deletions(-)

diff --git a/src/shapes/ImageGenShapeUtil.tsx b/src/shapes/ImageGenShapeUtil.tsx
index a61acbf..01bf2b1 100644
--- a/src/shapes/ImageGenShapeUtil.tsx
+++ b/src/shapes/ImageGenShapeUtil.tsx
@@ -804,6 +804,8 @@ export class ImageGenShape extends BaseBoxShapeUtil<IImageGen> {
                           }
                         }}
                         onPointerDown={(e) => e.stopPropagation()}
+                        onTouchStart={(e) => e.stopPropagation()}
+                        onTouchEnd={(e) => e.stopPropagation()}
                         style={{
                           flex: 1,
                           padding: '6px 10px',
@@ -819,6 +821,8 @@ export class ImageGenShape extends BaseBoxShapeUtil<IImageGen> {
                           justifyContent: 'center',
                           gap: '4px',
                           transition: 'background-color 0.15s',
+                          touchAction: 'manipulation',
+                          minHeight: '44px',
                         }}
                         onMouseEnter={(e) => (e.currentTarget.style.backgroundColor = '#f0f0f0')}
                         onMouseLeave={(e) => (e.currentTarget.style.backgroundColor = '#fff')}
@@ -850,6 +854,8 @@ export class ImageGenShape extends BaseBoxShapeUtil<IImageGen> {
                           console.log('✅ ImageGen: Image download initiated')
                         }}
                         onPointerDown={(e) => e.stopPropagation()}
+                        onTouchStart={(e) => e.stopPropagation()}
+                        onTouchEnd={(e) => e.stopPropagation()}
                         style={{
                           flex: 1,
                           padding: '6px 10px',
@@ -865,6 +871,8 @@ export class ImageGenShape extends BaseBoxShapeUtil<IImageGen> {
                           justifyContent: 'center',
                           gap: '4px',
                           transition: 'opacity 0.15s',
+                          touchAction: 'manipulation',
+                          minHeight: '44px',
                         }}
                         onMouseEnter={(e) => (e.currentTarget.style.opacity = '0.9')}
                         onMouseLeave={(e) => (e.currentTarget.style.opacity = '1')}
@@ -883,6 +891,8 @@ export class ImageGenShape extends BaseBoxShapeUtil<IImageGen> {
                           })
                         }}
                         onPointerDown={(e) => e.stopPropagation()}
+                        onTouchStart={(e) => e.stopPropagation()}
+                        onTouchEnd={(e) => e.stopPropagation()}
                         style={{
                           padding: '6px 10px',
                           backgroundColor: '#fff',
@@ -896,6 +906,9 @@ export class ImageGenShape extends BaseBoxShapeUtil<IImageGen> {
                           alignItems: 'center',
                           justifyContent: 'center',
                           transition: 'background-color 0.15s, color 0.15s',
+                          touchAction: 'manipulation',
+                          minWidth: '44px',
+                          minHeight: '44px',
                         }}
                         onMouseEnter={(e) => {
                           e.currentTarget.style.backgroundColor = '#fee'
@@ -952,6 +965,8 @@ export class ImageGenShape extends BaseBoxShapeUtil<IImageGen> {
                   borderRadius: "6px",
                   fontSize: 13,
                   padding: "0 10px",
+                  touchAction: "manipulation",
+                  minHeight: "44px",
                 }}
                 type="text"
                 placeholder="Enter image prompt..."
@@ -975,6 +990,9 @@ export class ImageGenShape extends BaseBoxShapeUtil<IImageGen> {
                 onPointerDown={(e) => {
                   e.stopPropagation()
                 }}
+                onTouchStart={(e) => {
+                  e.stopPropagation()
+                }}
                 onClick={(e) => {
                   e.stopPropagation()
                 }}
@@ -993,6 +1011,9 @@ export class ImageGenShape extends BaseBoxShapeUtil<IImageGen> {
                   fontWeight: "500",
                   fontSize: "13px",
                   opacity: shape.props.prompt.trim() && !shape.props.isLoading ? 1 : 0.6,
+                  touchAction: "manipulation",
+                  minWidth: "44px",
+                  minHeight: "44px",
                 }}
                 onPointerDown={(e) => {
                   e.stopPropagation()
@@ -1001,6 +1022,16 @@ export class ImageGenShape extends BaseBoxShapeUtil<IImageGen> {
                     handleGenerate()
                   }
                 }}
+                onTouchStart={(e) => {
+                  e.stopPropagation()
+                }}
+                onTouchEnd={(e) => {
+                  e.stopPropagation()
+                  e.preventDefault()
+                  if (shape.props.prompt.trim() && !shape.props.isLoading) {
+                    handleGenerate()
+                  }
+                }}
                 onClick={(e) => {
                   e.preventDefault()
                   e.stopPropagation()
@@ -1045,6 +1076,8 @@ export class ImageGenShape extends BaseBoxShapeUtil<IImageGen> {
                     })
                   }}
                   onPointerDown={(e) => e.stopPropagation()}
+                  onTouchStart={(e) => e.stopPropagation()}
+                  onTouchEnd={(e) => e.stopPropagation()}
                   style={{
                     padding: "2px 6px",
                     backgroundColor: "#fcc",
@@ -1053,6 +1086,9 @@ export class ImageGenShape extends BaseBoxShapeUtil<IImageGen> {
                     cursor: "pointer",
                     fontSize: "10px",
                     flexShrink: 0,
+                    touchAction: "manipulation",
+                    minWidth: "32px",
+                    minHeight: "32px",
                   }}
                 >
                   ✕
diff --git a/src/shapes/MultmuxShapeUtil.tsx b/src/shapes/MultmuxShapeUtil.tsx
index b89cf14..14aed66 100644
--- a/src/shapes/MultmuxShapeUtil.tsx
+++ b/src/shapes/MultmuxShapeUtil.tsx
@@ -569,8 +569,11 @@ export class MultmuxShape extends BaseBoxShapeUtil<IMultmuxShape> {
                     color: '#cdd6f4',
                     fontFamily: 'monospace',
                     fontSize: '14px',
+                    touchAction: 'manipulation',
+                    minHeight: '44px',
                   }}
                   onPointerDown={(e) => e.stopPropagation()}
+                  onTouchStart={(e) => e.stopPropagation()}
                   onMouseDown={(e) => e.stopPropagation()}
                 />
                 <button
@@ -586,8 +589,16 @@ export class MultmuxShape extends BaseBoxShapeUtil<IMultmuxShape> {
                     fontFamily: 'monospace',
                     fontSize: '16px',
                     transition: 'background-color 0.2s',
+                    touchAction: 'manipulation',
+                    minHeight: '44px',
                   }}
                   onPointerDown={(e) => e.stopPropagation()}
+                  onTouchStart={(e) => e.stopPropagation()}
+                  onTouchEnd={(e) => {
+                    e.stopPropagation()
+                    e.preventDefault()
+                    handleCreateSession()
+                  }}
                   onMouseDown={(e) => e.stopPropagation()}
                 >
                   + Create New Session
@@ -614,8 +625,16 @@ export class MultmuxShape extends BaseBoxShapeUtil<IMultmuxShape> {
                     color: '#cdd6f4',
                     cursor: 'pointer',
                     fontFamily: 'monospace',
+                    touchAction: 'manipulation',
+                    minHeight: '44px',
                   }}
                   onPointerDown={(e) => e.stopPropagation()}
+                  onTouchStart={(e) => e.stopPropagation()}
+                  onTouchEnd={(e) => {
+                    e.stopPropagation()
+                    e.preventDefault()
+                    if (!loadingSessions) fetchSessions()
+                  }}
                   onMouseDown={(e) => e.stopPropagation()}
                 >
                   {loadingSessions ? 'Loading...' : 'Refresh Sessions'}
@@ -648,8 +667,16 @@ export class MultmuxShape extends BaseBoxShapeUtil<IMultmuxShape> {
                           cursor: 'pointer',
                           textAlign: 'left',
                           fontFamily: 'monospace',
+                          touchAction: 'manipulation',
+                          minHeight: '44px',
                         }}
                         onPointerDown={(e) => e.stopPropagation()}
+                        onTouchStart={(e) => e.stopPropagation()}
+                        onTouchEnd={(e) => {
+                          e.stopPropagation()
+                          e.preventDefault()
+                          handleJoinSession(session.id)
+                        }}
                         onMouseDown={(e) => e.stopPropagation()}
                       >
                         <div style={{ fontWeight: 'bold' }}>{session.name}</div>
@@ -674,8 +701,16 @@ export class MultmuxShape extends BaseBoxShapeUtil<IMultmuxShape> {
                     fontSize: '12px',
                     fontFamily: 'monospace',
                     padding: '4px 0',
+                    touchAction: 'manipulation',
+                    minHeight: '44px',
                   }}
                   onPointerDown={(e) => e.stopPropagation()}
+                  onTouchStart={(e) => e.stopPropagation()}
+                  onTouchEnd={(e) => {
+                    e.stopPropagation()
+                    e.preventDefault()
+                    setShowAdvanced(!showAdvanced)
+                  }}
                   onMouseDown={(e) => e.stopPropagation()}
                 >
                   {showAdvanced ? '▼' : '▶'} Advanced Settings
@@ -704,8 +739,11 @@ export class MultmuxShape extends BaseBoxShapeUtil<IMultmuxShape> {
                           color: '#cdd6f4',
                           fontFamily: 'monospace',
                           fontSize: '12px',
+                          touchAction: 'manipulation',
+                          minHeight: '44px',
                         }}
                         onPointerDown={(e) => e.stopPropagation()}
+                        onTouchStart={(e) => e.stopPropagation()}
                         onMouseDown={(e) => e.stopPropagation()}
                       />
                     </label>
diff --git a/src/shapes/TranscriptionShapeUtil.tsx b/src/shapes/TranscriptionShapeUtil.tsx
index 10ec026..5247ca2 100644
--- a/src/shapes/TranscriptionShapeUtil.tsx
+++ b/src/shapes/TranscriptionShapeUtil.tsx
@@ -35,8 +35,9 @@ const AutoResizeTextarea: React.FC<{
   style: React.CSSProperties
   placeholder?: string
   onPointerDown?: (e: React.PointerEvent) => void
+  onTouchStart?: (e: React.TouchEvent) => void
   onWheel?: (e: React.WheelEvent) => void
-}> = ({ value, onChange, onBlur, onKeyDown, style, placeholder, onPointerDown, onWheel }) => {
+}> = ({ value, onChange, onBlur, onKeyDown, style, placeholder, onPointerDown, onTouchStart, onWheel }) => {
   const textareaRef = useRef<HTMLTextAreaElement>(null)
 
   useEffect(() => {
@@ -56,8 +57,12 @@ const AutoResizeTextarea: React.FC<{
       onBlur={onBlur}
       onKeyDown={onKeyDown}
       onPointerDown={onPointerDown}
+      onTouchStart={onTouchStart}
       onWheel={onWheel}
-      style={style}
+      style={{
+        ...style,
+        touchAction: 'manipulation',
+      }}
       placeholder={placeholder}
       autoFocus
     />
@@ -638,6 +643,9 @@ export class TranscriptionShape extends BaseBoxShapeUtil<ITranscription> {
       zIndex: 1000,
       position: 'relative',
       pointerEvents: 'auto', // Ensure button can receive clicks
+      touchAction: 'manipulation',
+      minWidth: '44px',
+      minHeight: '32px',
     }
 
     // Custom header content with status indicators and controls
@@ -665,6 +673,12 @@ export class TranscriptionShape extends BaseBoxShapeUtil<ITranscription> {
                 style={buttonStyle}
                 onClick={handleSaveEdit}
                 onPointerDown={(e) => e.stopPropagation()}
+                onTouchStart={(e) => e.stopPropagation()}
+                onTouchEnd={(e) => {
+                  e.stopPropagation()
+                  e.preventDefault()
+                  handleSaveEdit()
+                }}
               >
                 Save
               </button>
@@ -672,6 +686,12 @@ export class TranscriptionShape extends BaseBoxShapeUtil<ITranscription> {
                 style={buttonStyle}
                 onClick={handleCancelEdit}
                 onPointerDown={(e) => e.stopPropagation()}
+                onTouchStart={(e) => e.stopPropagation()}
+                onTouchEnd={(e) => {
+                  e.stopPropagation()
+                  e.preventDefault()
+                  handleCancelEdit()
+                }}
               >
                 Cancel
               </button>
@@ -723,6 +743,7 @@ export class TranscriptionShape extends BaseBoxShapeUtil<ITranscription> {
               style={textareaStyle}
               placeholder=""
               onPointerDown={(e) => e.stopPropagation()}
+              onTouchStart={(e) => e.stopPropagation()}
               onWheel={handleWheel}
             />
           ) : (
@@ -753,13 +774,13 @@ export class TranscriptionShape extends BaseBoxShapeUtil<ITranscription> {
             <button
               style={{
                 ...buttonStyle,
-                background: isRecording 
+                background: isRecording
                   ? "#ff4444"  // Red when recording
                   : isPaused
                   ? "#ffa500"  // Orange when paused
                   : (useWebSpeech ? webSpeechSupported : modelLoaded) ? "#007bff" : "#6c757d",  // Blue when ready to start, gray when loading
                 color: "white",
-                border: isRecording 
+                border: isRecording
                   ? "1px solid #cc0000"  // Red border when recording
                   : isPaused
                   ? "1px solid #cc8500"  // Orange border when paused
@@ -775,6 +796,16 @@ export class TranscriptionShape extends BaseBoxShapeUtil<ITranscription> {
               onPointerDown={(e) => {
                 e.stopPropagation()
               }}
+              onTouchStart={(e) => {
+                e.stopPropagation()
+              }}
+              onTouchEnd={(e) => {
+                e.stopPropagation()
+                e.preventDefault()
+                if (useWebSpeech ? webSpeechSupported : modelLoaded) {
+                  handleTranscriptionToggle()
+                }
+              }}
               disabled={useWebSpeech ? !webSpeechSupported : !modelLoaded}
               title={useWebSpeech ? (!webSpeechSupported ? "Web Speech API not supported" : "") : (!modelLoaded ? "Whisper model is loading - Please wait..." : "")}
             >
@@ -782,8 +813,8 @@ export class TranscriptionShape extends BaseBoxShapeUtil<ITranscription> {
                 if (isPaused) {
                   return "Resume"
                 }
-                const buttonText = isRecording 
-                  ? "Stop" 
+                const buttonText = isRecording
+                  ? "Stop"
                   : "Start"
                 return buttonText
               })()}
@@ -804,6 +835,14 @@ export class TranscriptionShape extends BaseBoxShapeUtil<ITranscription> {
                 onPointerDown={(e) => {
                   e.stopPropagation()
                 }}
+                onTouchStart={(e) => {
+                  e.stopPropagation()
+                }}
+                onTouchEnd={(e) => {
+                  e.stopPropagation()
+                  e.preventDefault()
+                  handlePauseToggle()
+                }}
                 title="Pause transcription"
               >
                 Pause
diff --git a/src/shapes/VideoGenShapeUtil.tsx b/src/shapes/VideoGenShapeUtil.tsx
index 75813f7..70db2e8 100644
--- a/src/shapes/VideoGenShapeUtil.tsx
+++ b/src/shapes/VideoGenShapeUtil.tsx
@@ -482,13 +482,15 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
                           setImageBase64('')
                         }}
                         onPointerDown={(e) => e.stopPropagation()}
+                        onTouchStart={(e) => e.stopPropagation()}
+                        onTouchEnd={(e) => e.stopPropagation()}
                         disabled={isGenerating}
                         style={{
                           position: 'absolute',
                           top: '4px',
                           right: '4px',
-                          width: '24px',
-                          height: '24px',
+                          width: '32px',
+                          height: '32px',
                           borderRadius: '50%',
                           border: 'none',
                           backgroundColor: 'rgba(0,0,0,0.6)',
@@ -497,7 +499,8 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
                           fontSize: '14px',
                           display: 'flex',
                           alignItems: 'center',
-                          justifyContent: 'center'
+                          justifyContent: 'center',
+                          touchAction: 'manipulation',
                         }}
                       >
                         ×
@@ -509,6 +512,12 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
                       <button
                         onClick={() => fileInputRef.current?.click()}
                         onPointerDown={(e) => e.stopPropagation()}
+                        onTouchStart={(e) => e.stopPropagation()}
+                        onTouchEnd={(e) => {
+                          e.stopPropagation()
+                          e.preventDefault()
+                          fileInputRef.current?.click()
+                        }}
                         disabled={isGenerating}
                         style={{
                           flex: 1,
@@ -522,7 +531,9 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
                           display: 'flex',
                           alignItems: 'center',
                           justifyContent: 'center',
-                          gap: '6px'
+                          gap: '6px',
+                          touchAction: 'manipulation',
+                          minHeight: '44px',
                         }}
                       >
                         📤 Upload Image
@@ -549,6 +560,7 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
                       placeholder="Or paste image URL..."
                       disabled={isGenerating}
                       onPointerDown={(e) => e.stopPropagation()}
+                      onTouchStart={(e) => e.stopPropagation()}
                       onMouseDown={(e) => e.stopPropagation()}
                       style={{
                         width: '100%',
@@ -558,7 +570,9 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
                         border: '1px solid #ddd',
                         borderRadius: '6px',
                         fontSize: '12px',
-                        boxSizing: 'border-box'
+                        boxSizing: 'border-box',
+                        touchAction: 'manipulation',
+                        minHeight: '44px',
                       }}
                     />
                   )}
@@ -578,6 +592,7 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
                     }
                     disabled={isGenerating}
                     onPointerDown={(e) => e.stopPropagation()}
+                    onTouchStart={(e) => e.stopPropagation()}
                     onMouseDown={(e) => e.stopPropagation()}
                     style={{
                       width: '100%',
@@ -590,7 +605,8 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
                       fontSize: '13px',
                       fontFamily: 'inherit',
                       resize: 'vertical',
-                      boxSizing: 'border-box'
+                      boxSizing: 'border-box',
+                      touchAction: 'manipulation',
                     }}
                   />
                 </div>
@@ -614,6 +630,7 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
                       }}
                       disabled={isGenerating}
                       onPointerDown={(e) => e.stopPropagation()}
+                      onTouchStart={(e) => e.stopPropagation()}
                       onMouseDown={(e) => e.stopPropagation()}
                       style={{
                         width: '100%',
@@ -623,7 +640,9 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
                         border: '1px solid #ddd',
                         borderRadius: '6px',
                         fontSize: '13px',
-                        boxSizing: 'border-box'
+                        boxSizing: 'border-box',
+                        touchAction: 'manipulation',
+                        minHeight: '44px',
                       }}
                     />
                   </div>
@@ -632,6 +651,14 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
                     onClick={handleGenerate}
                     disabled={isGenerating || !prompt.trim()}
                     onPointerDown={(e) => e.stopPropagation()}
+                    onTouchStart={(e) => e.stopPropagation()}
+                    onTouchEnd={(e) => {
+                      e.stopPropagation()
+                      e.preventDefault()
+                      if (!isGenerating && prompt.trim()) {
+                        handleGenerate()
+                      }
+                    }}
                     onMouseDown={(e) => e.stopPropagation()}
                     style={{
                       padding: '8px 20px',
@@ -644,7 +671,9 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
                       cursor: isGenerating ? 'not-allowed' : 'pointer',
                       transition: 'all 0.2s',
                       whiteSpace: 'nowrap',
-                      opacity: isGenerating || !prompt.trim() ? 0.6 : 1
+                      opacity: isGenerating || !prompt.trim() ? 0.6 : 1,
+                      touchAction: 'manipulation',
+                      minHeight: '44px',
                     }}
                   >
                     {isGenerating ? 'Generating...' : (mode === 'i2v' ? 'Animate Image' : 'Generate Video')}
@@ -697,14 +726,17 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
                   controls
                   autoPlay
                   loop
+                  playsInline
                   onPointerDown={(e) => e.stopPropagation()}
+                  onTouchStart={(e) => e.stopPropagation()}
                   onLoadedData={() => console.log('🎬 VideoGen: Video loaded successfully')}
                   onError={(e) => console.error('🎬 VideoGen: Video load error:', e)}
                   style={{
                     width: '100%',
                     maxHeight: '280px',
                     borderRadius: '6px',
-                    backgroundColor: '#000'
+                    backgroundColor: '#000',
+                    touchAction: 'manipulation',
                   }}
                 />
 
@@ -733,6 +765,8 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
                       })
                     }}
                     onPointerDown={(e) => e.stopPropagation()}
+                    onTouchStart={(e) => e.stopPropagation()}
+                    onTouchEnd={(e) => e.stopPropagation()}
                     onMouseDown={(e) => e.stopPropagation()}
                     style={{
                       flex: 1,
@@ -743,7 +777,9 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
                       borderRadius: '6px',
                       fontSize: '12px',
                       fontWeight: '500',
-                      cursor: 'pointer'
+                      cursor: 'pointer',
+                      touchAction: 'manipulation',
+                      minHeight: '44px',
                     }}
                   >
                     New Video
@@ -753,6 +789,8 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
                     href={videoUrl}
                     download="generated-video.mp4"
                     onPointerDown={(e) => e.stopPropagation()}
+                    onTouchStart={(e) => e.stopPropagation()}
+                    onTouchEnd={(e) => e.stopPropagation()}
                     onMouseDown={(e) => e.stopPropagation()}
                     style={{
                       flex: 1,
@@ -765,7 +803,12 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
                       fontWeight: '600',
                       textAlign: 'center',
                       textDecoration: 'none',
-                      cursor: 'pointer'
+                      cursor: 'pointer',
+                      touchAction: 'manipulation',
+                      minHeight: '44px',
+                      display: 'flex',
+                      alignItems: 'center',
+                      justifyContent: 'center',
                     }}
                   >
                     Download