feat(rmeets): live captions + MI internal-key auth
- Add Web Speech API captions overlay in the default Jitsi meeting view, broadcasting final/interim transcripts over Jitsi's data channel so each participant sees everyone's speech in real time. - Toggle via the MI FAB dropdown; degrades gracefully where SR is unsupported. - miApiFetch + /api/mi-proxy now forward X-MI-Internal-Key so the Meeting Intelligence recordings/search/meetings lists resolve from the backend without per-user tokens. - docker-compose exposes MEETING_INTELLIGENCE_API_URL, MI_INTERNAL_KEY, JITSI_URL to the rspace container. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
39fbd99897
commit
7cbf96de8b
|
|
@ -66,6 +66,9 @@ services:
|
||||||
- SCRIBUS_NOVNC_URL=https://design.rspace.online
|
- SCRIBUS_NOVNC_URL=https://design.rspace.online
|
||||||
- IPFS_API_URL=http://collab-server-ipfs-1:5001
|
- IPFS_API_URL=http://collab-server-ipfs-1:5001
|
||||||
- IPFS_GATEWAY_URL=https://ipfs.jeffemmett.com
|
- IPFS_GATEWAY_URL=https://ipfs.jeffemmett.com
|
||||||
|
- MEETING_INTELLIGENCE_API_URL=${MEETING_INTELLIGENCE_API_URL:-http://meeting-intelligence-api:8000}
|
||||||
|
- MI_INTERNAL_KEY=${MI_INTERNAL_KEY}
|
||||||
|
- JITSI_URL=${JITSI_URL:-https://jeffsi.localvibe.live}
|
||||||
depends_on:
|
depends_on:
|
||||||
rspace-db:
|
rspace-db:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
|
|
||||||
|
|
@ -35,6 +35,7 @@ function ensureMeetsDoc(space: string): MeetsDoc {
|
||||||
|
|
||||||
const JITSI_URL = process.env.JITSI_URL || "https://jeffsi.localvibe.live";
|
const JITSI_URL = process.env.JITSI_URL || "https://jeffsi.localvibe.live";
|
||||||
const MI_API_URL = process.env.MEETING_INTELLIGENCE_API_URL || "http://meeting-intelligence-api:8000";
|
const MI_API_URL = process.env.MEETING_INTELLIGENCE_API_URL || "http://meeting-intelligence-api:8000";
|
||||||
|
const MI_INTERNAL_KEY = process.env.MI_INTERNAL_KEY || "";
|
||||||
const routes = new Hono();
|
const routes = new Hono();
|
||||||
|
|
||||||
// ── Meeting Intelligence API helper ──
|
// ── Meeting Intelligence API helper ──
|
||||||
|
|
@ -43,10 +44,12 @@ async function miApiFetch(path: string, options?: { method?: string; body?: any
|
||||||
try {
|
try {
|
||||||
const controller = new AbortController();
|
const controller = new AbortController();
|
||||||
const timeout = setTimeout(() => controller.abort(), 8000);
|
const timeout = setTimeout(() => controller.abort(), 8000);
|
||||||
const fetchOpts: RequestInit = { signal: controller.signal };
|
const headers: Record<string, string> = {};
|
||||||
|
if (MI_INTERNAL_KEY) headers["X-MI-Internal-Key"] = MI_INTERNAL_KEY;
|
||||||
|
const fetchOpts: RequestInit = { signal: controller.signal, headers };
|
||||||
if (options?.method) fetchOpts.method = options.method;
|
if (options?.method) fetchOpts.method = options.method;
|
||||||
if (options?.body) {
|
if (options?.body) {
|
||||||
fetchOpts.headers = { "Content-Type": "application/json" };
|
headers["Content-Type"] = "application/json";
|
||||||
fetchOpts.body = JSON.stringify(options.body);
|
fetchOpts.body = JSON.stringify(options.body);
|
||||||
}
|
}
|
||||||
const res = await fetch(`${MI_API_URL}${path}`, fetchOpts);
|
const res = await fetch(`${MI_API_URL}${path}`, fetchOpts);
|
||||||
|
|
@ -419,9 +422,11 @@ routes.all("/api/mi-proxy/*", async (c) => {
|
||||||
try {
|
try {
|
||||||
const controller = new AbortController();
|
const controller = new AbortController();
|
||||||
const timeout = setTimeout(() => controller.abort(), 8000);
|
const timeout = setTimeout(() => controller.abort(), 8000);
|
||||||
|
const proxyHeaders: Record<string, string> = { "Content-Type": "application/json" };
|
||||||
|
if (MI_INTERNAL_KEY) proxyHeaders["X-MI-Internal-Key"] = MI_INTERNAL_KEY;
|
||||||
const upstream = await fetch(url.toString(), {
|
const upstream = await fetch(url.toString(), {
|
||||||
method: c.req.method,
|
method: c.req.method,
|
||||||
headers: { "Content-Type": "application/json" },
|
headers: proxyHeaders,
|
||||||
body: c.req.method !== "GET" && c.req.method !== "HEAD" ? await c.req.text() : undefined,
|
body: c.req.method !== "GET" && c.req.method !== "HEAD" ? await c.req.text() : undefined,
|
||||||
signal: controller.signal,
|
signal: controller.signal,
|
||||||
});
|
});
|
||||||
|
|
@ -701,6 +706,15 @@ routes.get("/:room", (c) => {
|
||||||
.mi-dropdown a:hover,.mi-dropdown button:hover{background:rgba(99,102,241,0.15)}
|
.mi-dropdown a:hover,.mi-dropdown button:hover{background:rgba(99,102,241,0.15)}
|
||||||
.mi-dropdown .mi-icon{font-size:1.1rem;width:22px;text-align:center;flex-shrink:0}
|
.mi-dropdown .mi-icon{font-size:1.1rem;width:22px;text-align:center;flex-shrink:0}
|
||||||
.mi-dropdown .mi-sep{height:1px;background:rgba(99,102,241,0.15);margin:4px 0}
|
.mi-dropdown .mi-sep{height:1px;background:rgba(99,102,241,0.15);margin:4px 0}
|
||||||
|
/* Live captions overlay */
|
||||||
|
#mi-captions{position:fixed;left:50%;bottom:92px;transform:translateX(-50%);z-index:9999;max-width:min(780px,94vw);display:flex;flex-direction:column;gap:4px;pointer-events:none;align-items:center}
|
||||||
|
#mi-captions.hidden{display:none}
|
||||||
|
.mi-cap-line{background:rgba(0,0,0,0.72);color:#f8fafc;padding:6px 14px;border-radius:10px;font-size:1rem;line-height:1.35;text-align:center;max-width:100%;backdrop-filter:blur(6px);box-shadow:0 2px 10px rgba(0,0,0,0.4);word-wrap:break-word}
|
||||||
|
.mi-cap-line .mi-cap-speaker{color:#a5b4fc;font-weight:600;margin-right:6px;font-size:0.85rem;text-transform:uppercase;letter-spacing:0.03em}
|
||||||
|
.mi-cap-line.interim{opacity:0.75;font-style:italic}
|
||||||
|
.mi-cap-status{position:fixed;left:12px;bottom:12px;z-index:10000;font-size:0.72rem;color:rgba(226,232,240,0.7);background:rgba(0,0,0,0.5);padding:4px 10px;border-radius:999px;pointer-events:none;max-width:80vw;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
||||||
|
.mi-cap-status.hidden{display:none}
|
||||||
|
@media(max-width:600px){#mi-captions{bottom:72px}.mi-cap-line{font-size:0.9rem;padding:5px 10px}}
|
||||||
</style>
|
</style>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
|
@ -710,6 +724,8 @@ routes.get("/:room", (c) => {
|
||||||
<div class="mi-fab">
|
<div class="mi-fab">
|
||||||
<button class="mi-fab-btn" id="mi-fab-toggle" title="Meeting Intelligence">🧠</button>
|
<button class="mi-fab-btn" id="mi-fab-toggle" title="Meeting Intelligence">🧠</button>
|
||||||
<div class="mi-dropdown" id="mi-dropdown">
|
<div class="mi-dropdown" id="mi-dropdown">
|
||||||
|
<button type="button" id="mi-cc-toggle"><span class="mi-icon">🗨</span> <span id="mi-cc-label">Turn on live captions</span></button>
|
||||||
|
<div class="mi-sep"></div>
|
||||||
<a href="${meetsBase}/meeting-intelligence"><span class="mi-icon">🧠</span> Meeting Intelligence</a>
|
<a href="${meetsBase}/meeting-intelligence"><span class="mi-icon">🧠</span> Meeting Intelligence</a>
|
||||||
<a href="${meetsBase}/recordings"><span class="mi-icon">🎥</span> Recordings</a>
|
<a href="${meetsBase}/recordings"><span class="mi-icon">🎥</span> Recordings</a>
|
||||||
<a href="${meetsBase}/search"><span class="mi-icon">🔍</span> Search Transcripts</a>
|
<a href="${meetsBase}/search"><span class="mi-icon">🔍</span> Search Transcripts</a>
|
||||||
|
|
@ -717,6 +733,8 @@ routes.get("/:room", (c) => {
|
||||||
<a href="${meetsBase}"><span class="mi-icon">🏠</span> rMeets Hub</a>
|
<a href="${meetsBase}"><span class="mi-icon">🏠</span> rMeets Hub</a>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
<div id="mi-captions" class="hidden"></div>
|
||||||
|
<div id="mi-cap-status" class="mi-cap-status hidden"></div>
|
||||||
<script src="${escapeHtml(JITSI_URL)}/libs/external_api.min.js"></script>
|
<script src="${escapeHtml(JITSI_URL)}/libs/external_api.min.js"></script>
|
||||||
<script>
|
<script>
|
||||||
// MI dropdown toggle
|
// MI dropdown toggle
|
||||||
|
|
@ -789,6 +807,159 @@ routes.get("/:room", (c) => {
|
||||||
+ '<a href="${meetsBase}/recordings">View Transcript & Summary</a>'
|
+ '<a href="${meetsBase}/recordings">View Transcript & Summary</a>'
|
||||||
+ '<a href="${meetsBase}">Back to rMeets</a></div>';
|
+ '<a href="${meetsBase}">Back to rMeets</a></div>';
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ── Live captions (Web Speech API, peer-broadcast via Jitsi data channel) ──
|
||||||
|
(function setupCaptions() {
|
||||||
|
var capsEl = document.getElementById("mi-captions");
|
||||||
|
var statusEl = document.getElementById("mi-cap-status");
|
||||||
|
var ccBtn = document.getElementById("mi-cc-toggle");
|
||||||
|
var ccLabel = document.getElementById("mi-cc-label");
|
||||||
|
var myName = "Me";
|
||||||
|
var myId = null;
|
||||||
|
var enabled = false;
|
||||||
|
var recognition = null;
|
||||||
|
var wantRunning = false;
|
||||||
|
var lines = []; // { id, speaker, text, interim, ts }
|
||||||
|
var lineSeq = 0;
|
||||||
|
var MAX_LINES = 3;
|
||||||
|
var FADE_MS = 6000;
|
||||||
|
|
||||||
|
var SR = window.SpeechRecognition || window.webkitSpeechRecognition;
|
||||||
|
if (!SR) {
|
||||||
|
ccLabel.textContent = "Captions not supported in this browser";
|
||||||
|
ccBtn.disabled = true;
|
||||||
|
ccBtn.style.opacity = "0.5";
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
api.addEventListener("videoConferenceJoined", function(e) {
|
||||||
|
myId = e.id;
|
||||||
|
if (e.displayName) myName = e.displayName;
|
||||||
|
});
|
||||||
|
api.addEventListener("displayNameChange", function(e) {
|
||||||
|
if (e.id === myId && e.displayname) myName = e.displayname;
|
||||||
|
});
|
||||||
|
|
||||||
|
function render() {
|
||||||
|
var now = Date.now();
|
||||||
|
lines = lines.filter(function(l) { return l.interim || (now - l.ts) < FADE_MS; });
|
||||||
|
capsEl.innerHTML = "";
|
||||||
|
var shown = lines.slice(-MAX_LINES);
|
||||||
|
shown.forEach(function(l) {
|
||||||
|
var d = document.createElement("div");
|
||||||
|
d.className = "mi-cap-line" + (l.interim ? " interim" : "");
|
||||||
|
d.innerHTML = '<span class="mi-cap-speaker">' + escapeText(l.speaker) + '</span>' + escapeText(l.text);
|
||||||
|
capsEl.appendChild(d);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
function escapeText(s) {
|
||||||
|
return String(s || '').replace(/[&<>"']/g, function(c) {
|
||||||
|
return { '&':'&', '<':'<', '>':'>', '"':'"', "'":''' }[c];
|
||||||
|
});
|
||||||
|
}
|
||||||
|
function pushLine(speaker, text, interim, senderKey) {
|
||||||
|
// Replace the most recent interim line from the same sender, otherwise append
|
||||||
|
var key = senderKey || speaker;
|
||||||
|
for (var i = lines.length - 1; i >= 0; i--) {
|
||||||
|
if (lines[i].interim && lines[i].senderKey === key) {
|
||||||
|
lines[i].text = text;
|
||||||
|
lines[i].interim = !!interim;
|
||||||
|
lines[i].ts = Date.now();
|
||||||
|
if (!interim) lines[i].interim = false;
|
||||||
|
render();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lines.push({ id: ++lineSeq, speaker: speaker, text: text, interim: !!interim, ts: Date.now(), senderKey: key });
|
||||||
|
if (lines.length > 20) lines = lines.slice(-20);
|
||||||
|
render();
|
||||||
|
}
|
||||||
|
setInterval(render, 1500);
|
||||||
|
|
||||||
|
function broadcast(text, isFinal) {
|
||||||
|
try {
|
||||||
|
var msg = JSON.stringify({ type: "caption", v: 1, final: !!isFinal, text: text });
|
||||||
|
api.executeCommand("sendEndpointTextMessage", "", msg);
|
||||||
|
} catch(e) { /* pre-join; ignore */ }
|
||||||
|
}
|
||||||
|
|
||||||
|
api.addEventListener("endpointTextMessageReceived", function(event) {
|
||||||
|
try {
|
||||||
|
var raw = event && event.data && event.data.eventData && event.data.eventData.text;
|
||||||
|
if (!raw) return;
|
||||||
|
var msg = JSON.parse(raw);
|
||||||
|
if (!msg || msg.type !== "caption" || !msg.text) return;
|
||||||
|
var sender = event.data.senderInfo || {};
|
||||||
|
var senderId = sender.id || sender.jid || "peer";
|
||||||
|
var speaker = "";
|
||||||
|
try { speaker = api.getDisplayName && api.getDisplayName(senderId); } catch(_) {}
|
||||||
|
if (!speaker) speaker = sender.displayName || "Guest";
|
||||||
|
pushLine(speaker, msg.text, !msg.final, "peer:" + senderId);
|
||||||
|
} catch(_) { /* ignore malformed */ }
|
||||||
|
});
|
||||||
|
|
||||||
|
function startRecognition() {
|
||||||
|
if (recognition) return;
|
||||||
|
recognition = new SR();
|
||||||
|
recognition.continuous = true;
|
||||||
|
recognition.interimResults = true;
|
||||||
|
recognition.lang = (navigator.language || "en-US");
|
||||||
|
recognition.onresult = function(evt) {
|
||||||
|
for (var i = evt.resultIndex; i < evt.results.length; i++) {
|
||||||
|
var res = evt.results[i];
|
||||||
|
var text = (res[0] && res[0].transcript || "").trim();
|
||||||
|
if (!text) continue;
|
||||||
|
var isFinal = !!res.isFinal;
|
||||||
|
pushLine(myName + " (you)", text, !isFinal, "me");
|
||||||
|
broadcast(text, isFinal);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
recognition.onerror = function(e) {
|
||||||
|
if (e.error === "no-speech" || e.error === "aborted") return;
|
||||||
|
setStatus("Captions paused: " + (e.error || "error"));
|
||||||
|
};
|
||||||
|
recognition.onend = function() {
|
||||||
|
recognition = null;
|
||||||
|
if (wantRunning) {
|
||||||
|
// Chrome stops after ~60s of continuous; restart
|
||||||
|
setTimeout(function() { if (wantRunning) startRecognition(); }, 300);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
try { recognition.start(); setStatus("Captions on — your voice is being shared"); }
|
||||||
|
catch(_) { /* already started — ignore */ }
|
||||||
|
}
|
||||||
|
|
||||||
|
function stopRecognition() {
|
||||||
|
wantRunning = false;
|
||||||
|
if (recognition) { try { recognition.stop(); } catch(_){} recognition = null; }
|
||||||
|
}
|
||||||
|
|
||||||
|
function setStatus(txt) {
|
||||||
|
if (!txt) { statusEl.classList.add("hidden"); statusEl.textContent = ""; return; }
|
||||||
|
statusEl.classList.remove("hidden");
|
||||||
|
statusEl.textContent = txt;
|
||||||
|
}
|
||||||
|
|
||||||
|
ccBtn.addEventListener("click", function(e) {
|
||||||
|
e.stopPropagation();
|
||||||
|
enabled = !enabled;
|
||||||
|
if (enabled) {
|
||||||
|
capsEl.classList.remove("hidden");
|
||||||
|
ccLabel.textContent = "Turn off live captions";
|
||||||
|
wantRunning = true;
|
||||||
|
startRecognition();
|
||||||
|
} else {
|
||||||
|
stopRecognition();
|
||||||
|
ccLabel.textContent = "Turn on live captions";
|
||||||
|
setStatus("");
|
||||||
|
capsEl.classList.add("hidden");
|
||||||
|
lines = [];
|
||||||
|
render();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
window.addEventListener("beforeunload", stopRecognition);
|
||||||
|
})();
|
||||||
} catch(e) {
|
} catch(e) {
|
||||||
document.getElementById("jitsi-container").innerHTML =
|
document.getElementById("jitsi-container").innerHTML =
|
||||||
'<div class="loading" style="color:#ef4444">Failed to connect: ' + e.message + '</div>';
|
'<div class="loading" style="color:#ef4444">Failed to connect: ' + e.message + '</div>';
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue