|
|
<!DOCTYPE html> |
|
|
<html> |
|
|
|
|
|
<head> |
|
|
<link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons"> |
|
|
<link rel="stylesheet" href="https://code.getmdl.io/1.3.0/material.indigo-pink.min.css"> |
|
|
<script defer src="https://code.getmdl.io/1.3.0/material.min.js"></script> |
|
|
|
|
|
<style> |
|
|
#videoElement { |
|
|
width: 640px; |
|
|
height: 480px; |
|
|
border-radius: 20px; |
|
|
} |
|
|
|
|
|
#canvasElement { |
|
|
display: none; |
|
|
width: 640px; |
|
|
height: 480px; |
|
|
} |
|
|
|
|
|
.demo-content { |
|
|
padding: 20px; |
|
|
display: flex; |
|
|
flex-direction: column; |
|
|
align-items: center; |
|
|
} |
|
|
|
|
|
.button-group { |
|
|
margin-bottom: 20px; |
|
|
} |
|
|
</style> |
|
|
</head> |
|
|
|
|
|
<body> |
|
|
<div class="mdl-layout mdl-js-layout mdl-layout--fixed-header"> |
|
|
<header class="mdl-layout__header"> |
|
|
<div class="mdl-layout__header-row"> |
|
|
|
|
|
<span class="mdl-layout-title">Gemini Live Demo</span> |
|
|
</div> |
|
|
</header> |
|
|
<main class="mdl-layout__content"> |
|
|
<div class="page-content"> |
|
|
<div class="demo-content"> |
|
|
|
|
|
<div class="button-group"> |
|
|
<button id="startButton" |
|
|
class="mdl-button mdl-js-button mdl-button--fab mdl-button--mini-fab mdl-button--colored"> |
|
|
<i class="material-icons">mic</i> |
|
|
</button> |
|
|
<button id="stopButton" |
|
|
class="mdl-button mdl-js-button mdl-button--fab mdl-button--mini-fab"> |
|
|
<i class="material-icons">mic_off</i> |
|
|
</button> |
|
|
</div> |
|
|
|
|
|
|
|
|
<video id="videoElement" autoplay style="width: 640px; height: 480px;"></video> |
|
|
|
|
|
|
|
|
<canvas id="canvasElement" style="width: 640px; height: 480px;"></canvas> |
|
|
|
|
|
<div id="chatLog"></div> |
|
|
</div> |
|
|
</div> |
|
|
</main> |
|
|
</div> |
|
|
|
|
|
<script defer> |
|
|
const URL = "ws://localhost:9083"; |
|
|
const video = document.getElementById("videoElement"); |
|
|
const canvas = document.getElementById("canvasElement"); |
|
|
let context; |
|
|
|
|
|
|
|
|
window.addEventListener("load", () => { |
|
|
context = canvas.getContext("2d"); |
|
|
setInterval(captureImage, 3000); |
|
|
}); |
|
|
|
|
|
const startButton = document.getElementById('startButton'); |
|
|
const stopButton = document.getElementById('stopButton'); |
|
|
let stream = null; |
|
|
let currentFrameB64; |
|
|
let webSocket = null; |
|
|
let audioContext = null; |
|
|
let mediaRecorder = null; |
|
|
let processor = null; |
|
|
let pcmData = []; |
|
|
let interval = null; |
|
|
let initialized = false; |
|
|
let audioInputContext; |
|
|
let workletNode; |
|
|
|
|
|
|
|
|
|
|
|
async function startScreenShare() { |
|
|
try { |
|
|
stream = await navigator.mediaDevices.getDisplayMedia({ |
|
|
video: { |
|
|
width: { max: 640 }, |
|
|
height: { max: 480 }, |
|
|
}, |
|
|
}); |
|
|
|
|
|
video.srcObject = stream; |
|
|
await new Promise(resolve => { |
|
|
video.onloadedmetadata = () => { |
|
|
console.log("video loaded metadata"); |
|
|
resolve(); |
|
|
} |
|
|
}); |
|
|
|
|
|
} catch (err) { |
|
|
console.error("Error accessing the screen: ", err); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
function captureImage() { |
|
|
if (stream && video.videoWidth > 0 && video.videoHeight > 0 && context) { |
|
|
canvas.width = 640; |
|
|
canvas.height = 480; |
|
|
context.drawImage(video, 0, 0, canvas.width, canvas.height); |
|
|
const imageData = canvas.toDataURL("image/jpeg").split(",")[1].trim(); |
|
|
currentFrameB64 = imageData; |
|
|
} |
|
|
else { |
|
|
console.log("no stream or video metadata not loaded"); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
window.addEventListener("load", async () => { |
|
|
await startScreenShare(); |
|
|
|
|
|
|
|
|
|
|
|
await initializeAudioContext(); |
|
|
|
|
|
connect(); |
|
|
}); |
|
|
|
|
|
function connect() { |
|
|
console.log("connecting: ", URL); |
|
|
|
|
|
webSocket = new WebSocket(URL); |
|
|
|
|
|
webSocket.onclose = (event) => { |
|
|
console.log("websocket closed: ", event); |
|
|
alert("Connection closed"); |
|
|
}; |
|
|
|
|
|
webSocket.onerror = (event) => { |
|
|
console.log("websocket error: ", event); |
|
|
}; |
|
|
|
|
|
webSocket.onopen = (event) => { |
|
|
console.log("websocket open: ", event); |
|
|
sendInitialSetupMessage(); |
|
|
}; |
|
|
|
|
|
webSocket.onmessage = receiveMessage; |
|
|
} |
|
|
|
|
|
function sendInitialSetupMessage() { |
|
|
|
|
|
console.log("sending setup message"); |
|
|
setup_client_message = { |
|
|
setup: { |
|
|
generation_config: { response_modalities: ["AUDIO"] }, |
|
|
}, |
|
|
}; |
|
|
|
|
|
webSocket.send(JSON.stringify(setup_client_message)); |
|
|
} |
|
|
|
|
|
|
|
|
function sendVoiceMessage(b64PCM) { |
|
|
if (webSocket == null) { |
|
|
console.log("websocket not initialized"); |
|
|
return; |
|
|
} |
|
|
|
|
|
payload = { |
|
|
realtime_input: { |
|
|
media_chunks: [{ |
|
|
mime_type: "audio/pcm", |
|
|
data: b64PCM, |
|
|
}, |
|
|
{ |
|
|
mime_type: "image/jpeg", |
|
|
data: currentFrameB64, |
|
|
}, |
|
|
], |
|
|
}, |
|
|
}; |
|
|
|
|
|
webSocket.send(JSON.stringify(payload)); |
|
|
console.log("sent: ", payload); |
|
|
} |
|
|
|
|
|
function receiveMessage(event) { |
|
|
const messageData = JSON.parse(event.data); |
|
|
const response = new Response(messageData); |
|
|
|
|
|
if (response.text) { |
|
|
displayMessage("GEMINI: " + response.text); |
|
|
} |
|
|
if (response.audioData) { |
|
|
injestAudioChuckToPlay(response.audioData); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
async function initializeAudioContext() { |
|
|
if (initialized) return; |
|
|
|
|
|
audioInputContext = new (window.AudioContext || |
|
|
window.webkitAudioContext)({ |
|
|
sampleRate: 24000 |
|
|
}); |
|
|
await audioInputContext.audioWorklet.addModule("pcm-processor.js"); |
|
|
workletNode = new AudioWorkletNode(audioInputContext, "pcm-processor"); |
|
|
workletNode.connect(audioInputContext.destination); |
|
|
initialized = true; |
|
|
} |
|
|
|
|
|
|
|
|
function base64ToArrayBuffer(base64) { |
|
|
const binaryString = window.atob(base64); |
|
|
const bytes = new Uint8Array(binaryString.length); |
|
|
for (let i = 0; i < binaryString.length; i++) { |
|
|
bytes[i] = binaryString.charCodeAt(i); |
|
|
} |
|
|
return bytes.buffer; |
|
|
} |
|
|
|
|
|
function convertPCM16LEToFloat32(pcmData) { |
|
|
const inputArray = new Int16Array(pcmData); |
|
|
const float32Array = new Float32Array(inputArray.length); |
|
|
|
|
|
for (let i = 0; i < inputArray.length; i++) { |
|
|
float32Array[i] = inputArray[i] / 32768; |
|
|
} |
|
|
|
|
|
return float32Array; |
|
|
} |
|
|
|
|
|
|
|
|
async function injestAudioChuckToPlay(base64AudioChunk) { |
|
|
try { |
|
|
if (audioInputContext.state === "suspended") { |
|
|
await audioInputContext.resume(); |
|
|
} |
|
|
const arrayBuffer = base64ToArrayBuffer(base64AudioChunk); |
|
|
const float32Data = convertPCM16LEToFloat32(arrayBuffer); |
|
|
|
|
|
workletNode.port.postMessage(float32Data); |
|
|
} catch (error) { |
|
|
console.error("Error processing audio chunk:", error); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
function recordChunk() { |
|
|
const buffer = new ArrayBuffer(pcmData.length * 2); |
|
|
const view = new DataView(buffer); |
|
|
pcmData.forEach((value, index) => { |
|
|
view.setInt16(index * 2, value, true); |
|
|
}); |
|
|
|
|
|
const base64 = btoa( |
|
|
String.fromCharCode.apply(null, new Uint8Array(buffer)) |
|
|
); |
|
|
|
|
|
sendVoiceMessage(base64); |
|
|
pcmData = []; |
|
|
} |
|
|
|
|
|
async function startAudioInput() { |
|
|
audioContext = new AudioContext({ |
|
|
sampleRate: 16000, |
|
|
}); |
|
|
|
|
|
const stream = await navigator.mediaDevices.getUserMedia({ |
|
|
audio: { |
|
|
channelCount: 1, |
|
|
sampleRate: 16000, |
|
|
}, |
|
|
}); |
|
|
|
|
|
const source = audioContext.createMediaStreamSource(stream); |
|
|
processor = audioContext.createScriptProcessor(4096, 1, 1); |
|
|
|
|
|
processor.onaudioprocess = (e) => { |
|
|
const inputData = e.inputBuffer.getChannelData(0); |
|
|
const pcm16 = new Int16Array(inputData.length); |
|
|
for (let i = 0; i < inputData.length; i++) { |
|
|
pcm16[i] = inputData[i] * 0x7fff; |
|
|
} |
|
|
pcmData.push(...pcm16); |
|
|
}; |
|
|
|
|
|
source.connect(processor); |
|
|
processor.connect(audioContext.destination); |
|
|
|
|
|
interval = setInterval(recordChunk, 3000); |
|
|
} |
|
|
|
|
|
function stopAudioInput() { |
|
|
if (processor) { |
|
|
processor.disconnect(); |
|
|
} |
|
|
if (audioContext) { |
|
|
audioContext.close(); |
|
|
} |
|
|
|
|
|
clearInterval(interval); |
|
|
} |
|
|
|
|
|
function displayMessage(message) { |
|
|
console.log(message); |
|
|
addParagraphToDiv("chatLog", message); |
|
|
} |
|
|
|
|
|
|
|
|
function addParagraphToDiv(divId, text) { |
|
|
const newParagraph = document.createElement("p"); |
|
|
newParagraph.textContent = text; |
|
|
const div = document.getElementById(divId); |
|
|
div.appendChild(newParagraph); |
|
|
} |
|
|
|
|
|
startButton.addEventListener('click', startAudioInput); |
|
|
stopButton.addEventListener('click', stopAudioInput); |
|
|
|
|
|
|
|
|
class Response { |
|
|
constructor(data) { |
|
|
this.text = null; |
|
|
this.audioData = null; |
|
|
this.endOfTurn = null; |
|
|
|
|
|
if (data.text) { |
|
|
this.text = data.text |
|
|
} |
|
|
|
|
|
if (data.audio) { |
|
|
this.audioData = data.audio; |
|
|
} |
|
|
} |
|
|
} |
|
|
</script> |
|
|
|
|
|
</body> |
|
|
|
|
|
</html> |
