feat(rmeets): live captions + MI internal-key auth
- Add Web Speech API captions overlay in the default Jitsi meeting view, broadcasting final/interim transcripts over Jitsi's data channel so each participant sees everyone's speech in real time. - Toggle via the MI FAB dropdown; degrades gracefully where SR is unsupported. - miApiFetch + /api/mi-proxy now forward X-MI-Internal-Key so the Meeting Intelligence recordings/search/meetings lists resolve from the backend without per-user tokens. - docker-compose exposes MEETING_INTELLIGENCE_API_URL, MI_INTERNAL_KEY, JITSI_URL to the rspace container. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
39fbd99897
commit
7cbf96de8b
|
|
@ -66,6 +66,9 @@ services:
|
|||
- SCRIBUS_NOVNC_URL=https://design.rspace.online
|
||||
- IPFS_API_URL=http://collab-server-ipfs-1:5001
|
||||
- IPFS_GATEWAY_URL=https://ipfs.jeffemmett.com
|
||||
- MEETING_INTELLIGENCE_API_URL=${MEETING_INTELLIGENCE_API_URL:-http://meeting-intelligence-api:8000}
|
||||
- MI_INTERNAL_KEY=${MI_INTERNAL_KEY}
|
||||
- JITSI_URL=${JITSI_URL:-https://jeffsi.localvibe.live}
|
||||
depends_on:
|
||||
rspace-db:
|
||||
condition: service_healthy
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ function ensureMeetsDoc(space: string): MeetsDoc {
|
|||
|
||||
const JITSI_URL = process.env.JITSI_URL || "https://jeffsi.localvibe.live";
|
||||
const MI_API_URL = process.env.MEETING_INTELLIGENCE_API_URL || "http://meeting-intelligence-api:8000";
|
||||
const MI_INTERNAL_KEY = process.env.MI_INTERNAL_KEY || "";
|
||||
const routes = new Hono();
|
||||
|
||||
// ── Meeting Intelligence API helper ──
|
||||
|
|
@ -43,10 +44,12 @@ async function miApiFetch(path: string, options?: { method?: string; body?: any
|
|||
try {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 8000);
|
||||
const fetchOpts: RequestInit = { signal: controller.signal };
|
||||
const headers: Record<string, string> = {};
|
||||
if (MI_INTERNAL_KEY) headers["X-MI-Internal-Key"] = MI_INTERNAL_KEY;
|
||||
const fetchOpts: RequestInit = { signal: controller.signal, headers };
|
||||
if (options?.method) fetchOpts.method = options.method;
|
||||
if (options?.body) {
|
||||
fetchOpts.headers = { "Content-Type": "application/json" };
|
||||
headers["Content-Type"] = "application/json";
|
||||
fetchOpts.body = JSON.stringify(options.body);
|
||||
}
|
||||
const res = await fetch(`${MI_API_URL}${path}`, fetchOpts);
|
||||
|
|
@ -419,9 +422,11 @@ routes.all("/api/mi-proxy/*", async (c) => {
|
|||
try {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 8000);
|
||||
const proxyHeaders: Record<string, string> = { "Content-Type": "application/json" };
|
||||
if (MI_INTERNAL_KEY) proxyHeaders["X-MI-Internal-Key"] = MI_INTERNAL_KEY;
|
||||
const upstream = await fetch(url.toString(), {
|
||||
method: c.req.method,
|
||||
headers: { "Content-Type": "application/json" },
|
||||
headers: proxyHeaders,
|
||||
body: c.req.method !== "GET" && c.req.method !== "HEAD" ? await c.req.text() : undefined,
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
|
@ -701,6 +706,15 @@ routes.get("/:room", (c) => {
|
|||
.mi-dropdown a:hover,.mi-dropdown button:hover{background:rgba(99,102,241,0.15)}
|
||||
.mi-dropdown .mi-icon{font-size:1.1rem;width:22px;text-align:center;flex-shrink:0}
|
||||
.mi-dropdown .mi-sep{height:1px;background:rgba(99,102,241,0.15);margin:4px 0}
|
||||
/* Live captions overlay */
|
||||
#mi-captions{position:fixed;left:50%;bottom:92px;transform:translateX(-50%);z-index:9999;max-width:min(780px,94vw);display:flex;flex-direction:column;gap:4px;pointer-events:none;align-items:center}
|
||||
#mi-captions.hidden{display:none}
|
||||
.mi-cap-line{background:rgba(0,0,0,0.72);color:#f8fafc;padding:6px 14px;border-radius:10px;font-size:1rem;line-height:1.35;text-align:center;max-width:100%;backdrop-filter:blur(6px);box-shadow:0 2px 10px rgba(0,0,0,0.4);word-wrap:break-word}
|
||||
.mi-cap-line .mi-cap-speaker{color:#a5b4fc;font-weight:600;margin-right:6px;font-size:0.85rem;text-transform:uppercase;letter-spacing:0.03em}
|
||||
.mi-cap-line.interim{opacity:0.75;font-style:italic}
|
||||
.mi-cap-status{position:fixed;left:12px;bottom:12px;z-index:10000;font-size:0.72rem;color:rgba(226,232,240,0.7);background:rgba(0,0,0,0.5);padding:4px 10px;border-radius:999px;pointer-events:none;max-width:80vw;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
||||
.mi-cap-status.hidden{display:none}
|
||||
@media(max-width:600px){#mi-captions{bottom:72px}.mi-cap-line{font-size:0.9rem;padding:5px 10px}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
|
@ -710,6 +724,8 @@ routes.get("/:room", (c) => {
|
|||
<div class="mi-fab">
|
||||
<button class="mi-fab-btn" id="mi-fab-toggle" title="Meeting Intelligence">🧠</button>
|
||||
<div class="mi-dropdown" id="mi-dropdown">
|
||||
<button type="button" id="mi-cc-toggle"><span class="mi-icon">🗨</span> <span id="mi-cc-label">Turn on live captions</span></button>
|
||||
<div class="mi-sep"></div>
|
||||
<a href="${meetsBase}/meeting-intelligence"><span class="mi-icon">🧠</span> Meeting Intelligence</a>
|
||||
<a href="${meetsBase}/recordings"><span class="mi-icon">🎥</span> Recordings</a>
|
||||
<a href="${meetsBase}/search"><span class="mi-icon">🔍</span> Search Transcripts</a>
|
||||
|
|
@ -717,6 +733,8 @@ routes.get("/:room", (c) => {
|
|||
<a href="${meetsBase}"><span class="mi-icon">🏠</span> rMeets Hub</a>
|
||||
</div>
|
||||
</div>
|
||||
<div id="mi-captions" class="hidden"></div>
|
||||
<div id="mi-cap-status" class="mi-cap-status hidden"></div>
|
||||
<script src="${escapeHtml(JITSI_URL)}/libs/external_api.min.js"></script>
|
||||
<script>
|
||||
// MI dropdown toggle
|
||||
|
|
@ -789,6 +807,159 @@ routes.get("/:room", (c) => {
|
|||
+ '<a href="${meetsBase}/recordings">View Transcript & Summary</a>'
|
||||
+ '<a href="${meetsBase}">Back to rMeets</a></div>';
|
||||
});
|
||||
|
||||
// ── Live captions (Web Speech API, peer-broadcast via Jitsi data channel) ──
|
||||
(function setupCaptions() {
|
||||
var capsEl = document.getElementById("mi-captions");
|
||||
var statusEl = document.getElementById("mi-cap-status");
|
||||
var ccBtn = document.getElementById("mi-cc-toggle");
|
||||
var ccLabel = document.getElementById("mi-cc-label");
|
||||
var myName = "Me";
|
||||
var myId = null;
|
||||
var enabled = false;
|
||||
var recognition = null;
|
||||
var wantRunning = false;
|
||||
var lines = []; // { id, speaker, text, interim, ts }
|
||||
var lineSeq = 0;
|
||||
var MAX_LINES = 3;
|
||||
var FADE_MS = 6000;
|
||||
|
||||
var SR = window.SpeechRecognition || window.webkitSpeechRecognition;
|
||||
if (!SR) {
|
||||
ccLabel.textContent = "Captions not supported in this browser";
|
||||
ccBtn.disabled = true;
|
||||
ccBtn.style.opacity = "0.5";
|
||||
return;
|
||||
}
|
||||
|
||||
api.addEventListener("videoConferenceJoined", function(e) {
|
||||
myId = e.id;
|
||||
if (e.displayName) myName = e.displayName;
|
||||
});
|
||||
api.addEventListener("displayNameChange", function(e) {
|
||||
if (e.id === myId && e.displayname) myName = e.displayname;
|
||||
});
|
||||
|
||||
function render() {
|
||||
var now = Date.now();
|
||||
lines = lines.filter(function(l) { return l.interim || (now - l.ts) < FADE_MS; });
|
||||
capsEl.innerHTML = "";
|
||||
var shown = lines.slice(-MAX_LINES);
|
||||
shown.forEach(function(l) {
|
||||
var d = document.createElement("div");
|
||||
d.className = "mi-cap-line" + (l.interim ? " interim" : "");
|
||||
d.innerHTML = '<span class="mi-cap-speaker">' + escapeText(l.speaker) + '</span>' + escapeText(l.text);
|
||||
capsEl.appendChild(d);
|
||||
});
|
||||
}
|
||||
function escapeText(s) {
|
||||
return String(s || '').replace(/[&<>"']/g, function(c) {
|
||||
return { '&':'&', '<':'<', '>':'>', '"':'"', "'":''' }[c];
|
||||
});
|
||||
}
|
||||
function pushLine(speaker, text, interim, senderKey) {
|
||||
// Replace the most recent interim line from the same sender, otherwise append
|
||||
var key = senderKey || speaker;
|
||||
for (var i = lines.length - 1; i >= 0; i--) {
|
||||
if (lines[i].interim && lines[i].senderKey === key) {
|
||||
lines[i].text = text;
|
||||
lines[i].interim = !!interim;
|
||||
lines[i].ts = Date.now();
|
||||
if (!interim) lines[i].interim = false;
|
||||
render();
|
||||
return;
|
||||
}
|
||||
}
|
||||
lines.push({ id: ++lineSeq, speaker: speaker, text: text, interim: !!interim, ts: Date.now(), senderKey: key });
|
||||
if (lines.length > 20) lines = lines.slice(-20);
|
||||
render();
|
||||
}
|
||||
setInterval(render, 1500);
|
||||
|
||||
function broadcast(text, isFinal) {
|
||||
try {
|
||||
var msg = JSON.stringify({ type: "caption", v: 1, final: !!isFinal, text: text });
|
||||
api.executeCommand("sendEndpointTextMessage", "", msg);
|
||||
} catch(e) { /* pre-join; ignore */ }
|
||||
}
|
||||
|
||||
api.addEventListener("endpointTextMessageReceived", function(event) {
|
||||
try {
|
||||
var raw = event && event.data && event.data.eventData && event.data.eventData.text;
|
||||
if (!raw) return;
|
||||
var msg = JSON.parse(raw);
|
||||
if (!msg || msg.type !== "caption" || !msg.text) return;
|
||||
var sender = event.data.senderInfo || {};
|
||||
var senderId = sender.id || sender.jid || "peer";
|
||||
var speaker = "";
|
||||
try { speaker = api.getDisplayName && api.getDisplayName(senderId); } catch(_) {}
|
||||
if (!speaker) speaker = sender.displayName || "Guest";
|
||||
pushLine(speaker, msg.text, !msg.final, "peer:" + senderId);
|
||||
} catch(_) { /* ignore malformed */ }
|
||||
});
|
||||
|
||||
function startRecognition() {
|
||||
if (recognition) return;
|
||||
recognition = new SR();
|
||||
recognition.continuous = true;
|
||||
recognition.interimResults = true;
|
||||
recognition.lang = (navigator.language || "en-US");
|
||||
recognition.onresult = function(evt) {
|
||||
for (var i = evt.resultIndex; i < evt.results.length; i++) {
|
||||
var res = evt.results[i];
|
||||
var text = (res[0] && res[0].transcript || "").trim();
|
||||
if (!text) continue;
|
||||
var isFinal = !!res.isFinal;
|
||||
pushLine(myName + " (you)", text, !isFinal, "me");
|
||||
broadcast(text, isFinal);
|
||||
}
|
||||
};
|
||||
recognition.onerror = function(e) {
|
||||
if (e.error === "no-speech" || e.error === "aborted") return;
|
||||
setStatus("Captions paused: " + (e.error || "error"));
|
||||
};
|
||||
recognition.onend = function() {
|
||||
recognition = null;
|
||||
if (wantRunning) {
|
||||
// Chrome stops after ~60s of continuous; restart
|
||||
setTimeout(function() { if (wantRunning) startRecognition(); }, 300);
|
||||
}
|
||||
};
|
||||
try { recognition.start(); setStatus("Captions on — your voice is being shared"); }
|
||||
catch(_) { /* already started — ignore */ }
|
||||
}
|
||||
|
||||
function stopRecognition() {
|
||||
wantRunning = false;
|
||||
if (recognition) { try { recognition.stop(); } catch(_){} recognition = null; }
|
||||
}
|
||||
|
||||
function setStatus(txt) {
|
||||
if (!txt) { statusEl.classList.add("hidden"); statusEl.textContent = ""; return; }
|
||||
statusEl.classList.remove("hidden");
|
||||
statusEl.textContent = txt;
|
||||
}
|
||||
|
||||
ccBtn.addEventListener("click", function(e) {
|
||||
e.stopPropagation();
|
||||
enabled = !enabled;
|
||||
if (enabled) {
|
||||
capsEl.classList.remove("hidden");
|
||||
ccLabel.textContent = "Turn off live captions";
|
||||
wantRunning = true;
|
||||
startRecognition();
|
||||
} else {
|
||||
stopRecognition();
|
||||
ccLabel.textContent = "Turn on live captions";
|
||||
setStatus("");
|
||||
capsEl.classList.add("hidden");
|
||||
lines = [];
|
||||
render();
|
||||
}
|
||||
});
|
||||
|
||||
window.addEventListener("beforeunload", stopRecognition);
|
||||
})();
|
||||
} catch(e) {
|
||||
document.getElementById("jitsi-container").innerHTML =
|
||||
'<div class="loading" style="color:#ef4444">Failed to connect: ' + e.message + '</div>';
|
||||
|
|
|
|||
Loading…
Reference in New Issue