File size: 5,633 Bytes
774c14a 814dd9f 774c14a 59704f8 774c14a 4c13885 774c14a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 |
import streamlit as st
import streamlit.components.v1 as components
from transformers import AutoTokenizer, AutoModelForCausalLM
# Set page configuration
st.set_page_config(page_title="First Conscious Quadrant with Detokenizer", layout="wide")
# Title
st.title("First Conscious Quadrant with Detokenizer")
# HTML content (your original HTML/JS code)
html_content = """
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Interactive Base 50256 Grid</title>
<style>
body {
font-family: Arial, sans-serif;
display: flex;
justify-content: center;
align-items: center;
height: 100vh;
margin: 0;
background-color: #f0f0f0;
}
.container {
text-align: center;
}
#grid {
max-width: 80vmin;
max-height: 80vmin;
border: 1px solid #ccc;
}
.output {
margin-top: 20px;
font-size: 18px;
font-weight: bold;
}
</style>
</head>
<body>
<div class="container">
<canvas id="grid" width="1000" height="1000"></canvas>
<div id="clickedOutput" class="output">Click on the grid to select a coordinate</div>
<div id="hoverOutput">Hover Coordinate: (X: 0, Y: 0)</div>
</div>
<script>
const canvas = document.getElementById('grid');
const ctx = canvas.getContext('2d');
const clickedOutput = document.getElementById('clickedOutput');
const hoverOutput = document.getElementById('hoverOutput');
const gridSizeX = 50255;
const gridSizeY = 50255;
const cellSizeX = canvas.width / 16;
const cellSizeY = canvas.height / 16;
function drawGrid() {
ctx.fillStyle = 'white';
ctx.fillRect(0, 0, canvas.width, canvas.height);
ctx.strokeStyle = '#ccc';
ctx.lineWidth = 1;
for (let i = cellSizeX; i < canvas.width; i += cellSizeX) {
ctx.beginPath();
ctx.moveTo(i, 0);
ctx.lineTo(i, canvas.height);
ctx.stroke();
}
for (let i = cellSizeY; i < canvas.height; i += cellSizeY) {
ctx.beginPath();
ctx.moveTo(0, i);
ctx.lineTo(canvas.width, i);
ctx.stroke();
}
ctx.fillStyle = 'black';
ctx.font = '16px Arial';
ctx.fillText('0,0', 5, canvas.height - 5);
ctx.fillText(`${gridSizeX},0`, canvas.width - 60, canvas.height - 5);
ctx.fillText(`0,${gridSizeY}`, 5, 20);
ctx.fillText(`${gridSizeX},${gridSizeY}`, canvas.width - 100, 20);
}
function getCoordinates(event) {
const rect = canvas.getBoundingClientRect();
const x = Math.min(Math.floor((event.clientX - rect.left) / rect.width * gridSizeX), gridSizeX);
const y = Math.min(gridSizeY - Math.floor((event.clientY - rect.top) / rect.height * gridSizeY), gridSizeY);
return { x, y };
}
canvas.addEventListener('mousemove', (event) => {
const { x, y } = getCoordinates(event);
hoverOutput.textContent = `Hover Coordinate: (X: ${x}, Y: ${y})`;
});
canvas.addEventListener('click', (event) => {
const { x, y } = getCoordinates(event);
const combinedCoord = x * 100000 + y;
clickedOutput.textContent = `Clicked Coordinate: ${combinedCoord.toString().padStart(10, '0')}`;
window.parent.postMessage({type: 'clickedCoordinate', value: combinedCoord.toString().padStart(10, '0')}, '*');
});
canvas.addEventListener('mouseleave', () => {
hoverOutput.textContent = 'Hover Coordinate: (X: 0, Y: 0)';
});
drawGrid();
</script>
</body>
</html>
"""
# Embed the HTML content
components.html(html_content, height=700, scrolling=True)
# Load the tokenizer
tokenizer = AutoTokenizer.from_pretrained('gpt2')
# Detokenization section
st.header("Detokenization")
token_ids = st.text_input("Enter token IDs (concatenated without spaces):", "")
def split_token_ids(concatenated_ids, length=5):
return [concatenated_ids[i:i+length] for i in range(0, len(concatenated_ids), length)]
def remove_leading_zeros(grouped_ids):
return [id.lstrip('0') for id in grouped_ids]
if st.button("Detokenize"):
split_ids = split_token_ids(token_ids)
cleaned_ids = remove_leading_zeros(split_ids)
cleaned_token_ids_str = ' '.join(cleaned_ids)
token_id_list = [int(id) for id in cleaned_ids if id.isdigit()]
detokenized_sentence = tokenizer.decode(token_id_list)
st.write("Grouped and cleaned token IDs:")
st.write(cleaned_token_ids_str)
st.write("Detokenized sentence:")
st.write(detokenized_sentence)
# Load the model
gpt2 = AutoModelForCausalLM.from_pretrained('gpt2')
# Display help for the GPT-2 model
if st.checkbox("Show GPT-2 Model Help"):
st.write("Help GPT2")
st.help(gpt2)
# JavaScript to handle messages from the iframe
components.html(
"""
<script>
window.addEventListener('message', function(event) {
if (event.data.type === 'clickedCoordinate') {
document.querySelector('input[aria-label="Enter token IDs (concatenated without spaces):"]').value = event.data.value;
document.querySelector('button[kind="secondary"]').click();
}
}, false);
</script>
"""
) |