Spaces:

jbilcke-hf
/

GenCity

Running

File size: 27,425 Bytes

<!DOCTYPE html>
<html>
<head>
    <title>Text to Terrain Generator</title>
    <style>
        body { margin: 0; }
        canvas { display: block; }
        #prompt-input {
            position: absolute;
            top: 10px;
            left: 10px;
            padding: 8px;
            width: 200px;
            border: 1px solid #ccc;
            border-radius: 4px;
            font-size: 14px;
        }
        #camera-toggle {
            position: absolute;
            top: 10px;
            right: 10px;
            padding: 8px 16px;
            background-color: #3498db;
            color: white;
            border: none;
            border-radius: 4px;
            cursor: pointer;
            font-size: 14px;
            transition: background-color 0.3s;
        }
        #camera-toggle:hover {
            background-color: #2980b9;
        }
        #heightmap {
            display: none;
        }
        #loadingOverlay {
            display: none;
            position: fixed;
            top: 0;
            left: 0;
            width: 100%;
            height: 100%;
            background-color: rgba(0, 0, 0, 0.7);
            z-index: 9999;
            justify-content: center;
            align-items: center;
        }
        .loader {
            border: 5px solid #f3f3f3;
            border-top: 5px solid #3498db;
            border-radius: 50%;
            width: 50px;
            height: 50px;
            animation: spin 1s linear infinite;
        }
        @keyframes spin {
            0% { transform: rotate(0deg); }
            100% { transform: rotate(360deg); }
        }
    </style>
</head>
<body>
    <input type="text" id="prompt-input" placeholder="Countryside house with trees..">
    <button id="camera-toggle">Toggle Camera</button>
    <div id="loadingOverlay">
        <div class="loader"></div>
    </div>
    <canvas id="heightmap" width="256" height="256"></canvas>
    <script type="importmap">
        {
            "imports": {
                "three": "https://cdnjs.cloudflare.com/ajax/libs/three.js/0.172.0/three.module.min.js",
                "three/addons/": "https://unpkg.com/[email protected]/examples/jsm/",
                "three/examples/": "https://unpkg.com/[email protected]/examples/jsm/",
                "@gradio/client": "https://cdn.jsdelivr.net/npm/@gradio/[email protected]/+esm"
            }
        }
    </script>
    <script type="module">
        import * as THREE from 'three';
        import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
        import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
        import { Client } from "@gradio/client";


        let isLoading = false;
        let lastPromptInputValue = "";
        let isDragging = false;
        let mouseDownPosition = new THREE.Vector2();
        const objectCountByType = new Map();
        let terrainInstances;
        let objectInstances = new Map();
        const TILE_SIZE = 4;
        const GRID_SIZE = 10;
        let currentTileType = null;
        let hoveredTileIndex = -1;
        const raycaster = new THREE.Raycaster();
        const mouse = new THREE.Vector2();
        let highlightMesh = null;
        const tileTypes = new Map();
        const gltfLoader = new GLTFLoader();

        // Scene setup
        const scene = new THREE.Scene();
        scene.background = new THREE.Color(0x87CEEB);

        // Camera setup with both perspectives
        const frustumSize = 40;
        const aspect = window.innerWidth / window.innerHeight;
        
        const orthoCamera = new THREE.OrthographicCamera(
            -20, 20, 20, -20,
            0.1, 2000
        );
        orthoCamera.position.set(20, 20, -20);
        orthoCamera.lookAt(0, 0, 0);

        const perspCamera = new THREE.PerspectiveCamera(
            75, window.innerWidth / window.innerHeight, 0.1, 2000
        );
        perspCamera.position.set(20, 20, -20);
        perspCamera.lookAt(0, 0, 0);

        let currentCamera = orthoCamera;
        let isOrthographic = true;

        // Renderer setup
        const renderer = new THREE.WebGLRenderer({ antialias: true });
        renderer.setSize(window.innerWidth, window.innerHeight);
        renderer.shadowMap.enabled = true;
        renderer.shadowMap.type = THREE.PCFSoftShadowMap;
        document.body.appendChild(renderer.domElement);

        // Controls
        let controls = new OrbitControls(currentCamera, renderer.domElement);
        controls.enableDamping = true;

       // Lighting
        const directionalLight = new THREE.DirectionalLight(0xffffff, 4.5); // Increased from 1.5 to 4.5
        directionalLight.position.set(20, 30, 20);
        directionalLight.castShadow = true;

        // Improve shadow quality
        directionalLight.shadow.mapSize.width = 2048;
        directionalLight.shadow.mapSize.height = 2048;
        directionalLight.shadow.camera.near = 0.1;
        directionalLight.shadow.camera.far = 100;
        directionalLight.shadow.camera.left = -30;
        directionalLight.shadow.camera.right = 30;
        directionalLight.shadow.camera.top = 30;
        directionalLight.shadow.camera.bottom = -30;
        directionalLight.shadow.bias = -0.001;

        // Add a brighter ambient light
        scene.add(new THREE.AmbientLight(0x404040, 1.2));

        // Add lighter fog
        scene.fog = new THREE.FogExp2(0x87CEEB, 0.015);

        // Enable shadow mapping in the renderer
        renderer.shadowMap.enabled = true;
        renderer.shadowMap.type = THREE.PCFSoftShadowMap;

        scene.add(directionalLight);


        // Height map generation
        const canvasH = document.getElementById('heightmap');
        const ctx = canvasH.getContext('2d');
        const heightMap = new THREE.CanvasTexture(canvasH);

        function createSeamlessNoise(size, octaves) {
            const noise = new Array(size * size).fill(0);
            
            function smoothStep(t) {
                return t * t * (3 - 2 * t);
            }
            
            function interpolate(a, b, t) {
                return a + (b - a) * smoothStep(t);
            }
            
            for (let octave = 0; octave < octaves; octave++) {
                const frequency = 1 << octave;
                const amplitude = 1 / (1 << octave);
                
                const grid = new Array((frequency + 1) * (frequency + 1));
                for (let i = 0; i <= frequency; i++) {
                    for (let j = 0; j <= frequency; j++) {
                        grid[i * (frequency + 1) + j] = Math.random();
                    }
                }
                
                for (let i = 0; i <= frequency; i++) {
                    grid[i * (frequency + 1) + frequency] = grid[i * (frequency + 1)];
                }
                for (let j = 0; j <= frequency; j++) {
                    grid[frequency * (frequency + 1) + j] = grid[j];
                }
                grid[frequency * (frequency + 1) + frequency] = grid[0];
                
                const cellSize = size / frequency;
                for (let y = 0; y < size; y++) {
                    for (let x = 0; x < size; x++) {
                        const gridX = Math.floor(x / cellSize);
                        const gridY = Math.floor(y / cellSize);
                        const fracX = (x % cellSize) / cellSize;
                        const fracY = (y % cellSize) / cellSize;
                        
                        const v1 = grid[gridY * (frequency + 1) + gridX];
                        const v2 = grid[gridY * (frequency + 1) + (gridX + 1)];
                        const v3 = grid[(gridY + 1) * (frequency + 1) + gridX];
                        const v4 = grid[(gridY + 1) * (frequency + 1) + (gridX + 1)];
                        
                        const i1 = interpolate(v1, v2, fracX);
                        const i2 = interpolate(v3, v4, fracX);
                        const value = interpolate(i1, i2, fracY);
                        
                        noise[y * size + x] += value * amplitude;
                    }
                }
            }
            
            return noise;
        }

        function createHeightMap() {
            const size = 256;
            const noise = createSeamlessNoise(size, 8);
            
            const imageData = ctx.createImageData(size, size);
            let min = Infinity, max = -Infinity;
            
            for (let i = 0; i < noise.length; i++) {
                min = Math.min(min, noise[i]);
                max = Math.max(max, noise[i]);
            }
            
            for (let i = 0; i < noise.length; i++) {
                const normalized = Math.floor(((noise[i] - min) / (max - min)) * 255);
                imageData.data[i * 4] = normalized;
                imageData.data[i * 4 + 1] = normalized;
                imageData.data[i * 4 + 2] = normalized;
                imageData.data[i * 4 + 3] = 255;
            }
            
            ctx.putImageData(imageData, 0, 0);
            heightMap.needsUpdate = true;
            
            if (terrainInstances?.material) {
                terrainInstances.material.displacementMap = heightMap;
                terrainInstances.material.needsUpdate = true;
            }
        }

        // Create base terrain type
        function createBaseTileType() {
            const geometry = new THREE.PlaneGeometry(TILE_SIZE, TILE_SIZE, 64, 64);
            const material = new THREE.MeshStandardMaterial({
                color: 0x46732a,
                displacementMap: heightMap,
                displacementScale: 0.8,
                flatShading: false,
                side: THREE.DoubleSide,
                shadowSide: THREE.DoubleSide
            });
            
            tileTypes.set('base', { geometry, material });
        }

        function createHighlightMesh() {
            const geometry = new THREE.PlaneGeometry(TILE_SIZE, TILE_SIZE);
            const edges = new THREE.EdgesGeometry(geometry);
            const material = new THREE.LineBasicMaterial({ 
                color: 0xffff00,
                linewidth: 2,
                transparent: true,
                opacity: 0.9
            });
            
            highlightMesh = new THREE.LineSegments(edges, material);
            highlightMesh.rotation.x = -Math.PI / 2;
            highlightMesh.visible = false;
            scene.add(highlightMesh);
        }

        function initializeTerrain() {
            createBaseTileType();
            createHighlightMesh(); 
                    
            const baseTile = tileTypes.get('base');
            const instanceCount = GRID_SIZE * GRID_SIZE;
            terrainInstances = new THREE.InstancedMesh(
                baseTile.geometry,
                baseTile.material,
                instanceCount
            );
            terrainInstances.castShadow = true;
            terrainInstances.receiveShadow = true;

            const matrix = new THREE.Matrix4();
            const position = new THREE.Vector3();
            const rotation = new THREE.Euler();
            const quaternion = new THREE.Quaternion();
            const scale = new THREE.Vector3(1, 1, 1);

            let index = 0;
            const offset = (GRID_SIZE * TILE_SIZE) / 2 - TILE_SIZE / 2;
            
            for (let x = 0; x < GRID_SIZE; x++) {
                for (let z = 0; z < GRID_SIZE; z++) {
                    position.set(
                        x * TILE_SIZE - offset,
                        0,
                        z * TILE_SIZE - offset
                    );
                    rotation.set(-Math.PI / 2, 0, 0);
                    quaternion.setFromEuler(rotation);
                    
                    matrix.compose(position, quaternion, scale);
                    terrainInstances.setMatrixAt(index, matrix);
                    index++;
                }
            }

            scene.add(terrainInstances);
        }

        async function createGLBTileType(modelUrl, typeName) {
            return new Promise((resolve, reject) => {
                gltfLoader.load(modelUrl, (gltf) => {
                    const model = gltf.scene;
                    
                    const box = new THREE.Box3().setFromObject(model);
                    const size = box.getSize(new THREE.Vector3());

                    const geometry = new THREE.BoxGeometry(1, 1, 1);
                    
                    const material = new THREE.MeshStandardMaterial({ visible: false });
                    
                    tileTypes.set(typeName, {
                        geometry,
                        material,
                        model: model.clone()
                    });
                    
                    resolve();
                }, undefined, reject);
            });
        }

             
        // First, create a function to find the Y position of the widest cross-section
        function findWidestCrossSection(mesh) {
        // Track the maximum width/depth we find and its Y position
        let maxArea = 0;
        let maxAreaY = 0;
        
        // Get all vertices from the mesh and its children
        const vertices = [];
        mesh.traverse((child) => {
        if (child.isMesh && child.geometry) {
            const positions = child.geometry.attributes.position;
            const vertexCount = positions.count;
            
            // Transform vertices to world space
            const matrix = child.matrixWorld;
            for (let i = 0; i < vertexCount; i++) {
            const vertex = new THREE.Vector3();
            vertex.fromBufferAttribute(positions, i);
            vertex.applyMatrix4(matrix);
            vertices.push(vertex);
            }
        }
        });

        if (vertices.length === 0) return 0;

        // Find Y range to analyze
        const yValues = vertices.map(v => v.y);
        const minY = Math.min(...yValues);
        const maxY = Math.max(...yValues);
        
        // Sample Y positions at regular intervals
        const steps = 128; // Number of cross-sections to check
        const yStep = (maxY - minY) / steps;
        
        for (let i = 0; i <= steps; i++) {
        const currentY = minY + (i * yStep);
        
        // Find vertices near this Y level (within small threshold)
        const threshold = yStep / 2;
        const sectionVertices = vertices.filter(v => 
            Math.abs(v.y - currentY) < threshold
        );
        
        if (sectionVertices.length > 0) {
            // Calculate bounding area of this cross-section
            const xValues = sectionVertices.map(v => v.x);
            const zValues = sectionVertices.map(v => v.z);
            
            const width = Math.max(...xValues) - Math.min(...xValues);
            const depth = Math.max(...zValues) - Math.min(...zValues);
            const area = width * depth;
            
            if (area > maxArea) {
            maxArea = area;
            maxAreaY = currentY;
            }
        }
        }
        
        return maxAreaY;
    }

    
    function replaceTileInstance(instanceIndex, newTypeName) {
        const newType = tileTypes.get(newTypeName);
        if (!newType) return;
        
        // Get original transformation matrix from terrain instance
        const matrix = new THREE.Matrix4();
        terrainInstances.getMatrixAt(instanceIndex, matrix);
        
        // Extract position from matrix
        const position = new THREE.Vector3();
        const rotation = new THREE.Quaternion();
        const scale = new THREE.Vector3();
        matrix.decompose(position, rotation, scale);
        
        // Check if there's an existing object of the same type
        if (objectInstances.has(instanceIndex)) {
            const existing = objectInstances.get(instanceIndex);
            if (existing.typeName === newTypeName) {
                // Rotate existing object's rotation group by 90 degrees
                const currentRotationY = (existing.rotationY || 0) + Math.PI / 2;
                const normalizedRotation = currentRotationY % (Math.PI * 2);
                
                existing.rotationGroup.rotation.y = (Math.PI / 4) + normalizedRotation;
                existing.rotationY = normalizedRotation;
                return; // Exit early as we just rotated the existing object
            } else {
                // Remove existing object if it's a different type
                scene.remove(existing.rotationGroup);
            }
        }
        
        // Create new mesh from model
        const newMesh = newType.model.clone();

        // Create hierarchy of groups for different transformations
        const rotationGroup = new THREE.Group();    // Handles user-controlled Y rotation
        const orientationGroup = new THREE.Group(); // Handles initial orientation
        
        // Build hierarchy
        scene.add(rotationGroup);
        rotationGroup.add(orientationGroup);
        orientationGroup.add(newMesh);

        // Position the top-level group at the tile location
        rotationGroup.position.copy(position);

        // Enable shadows
        newMesh.traverse((child) => {
            if (child.isMesh) {
                child.castShadow = true;
                child.receiveShadow = true;
            }
            if (child.material) {
                child.material.shadowSide = THREE.DoubleSide;
                child.material.needsUpdate = true;
            }
        });
        
  
        // Create a bounding box to calculate the mesh's dimensions
        const bbox = new THREE.Box3().setFromObject(newMesh);
        const meshCenter = bbox.getCenter(new THREE.Vector3());
        
        // Center the mesh on its local origin
        newMesh.position.sub(meshCenter);
        
        // Apply initial orientation using clean, separate rotations
        orientationGroup.rotation.x = 0.6; // fix mesh generation tiling
        orientationGroup.rotation.y = 0;
        orientationGroup.rotation.z = 0;
        
        // Find the Y position of widest cross-section and adjust position
        const baseY = findWidestCrossSection(newMesh);
        newMesh.position.y -= baseY;
        newMesh.position.y += 1.2;

        rotationGroup.rotation.y = Math.PI / 4;

             // Apply scale to the mesh
             newMesh.scale.set(5.6, 5.6, 5.6);
        
        // Adjust XZ position to ensure centering after rotation
        bbox.setFromObject(newMesh);
        const rotatedCenter = bbox.getCenter(new THREE.Vector3());
        //newMesh.position.x -= 1.0;//rotatedCenter.x;
        //newMesh.position.z -= 0;//rotatedCenter.z;

   
        // Add new object to scene and track it with metadata
        objectInstances.set(instanceIndex, {
            rotationGroup: rotationGroup,
            orientationGroup: orientationGroup,
            mesh: newMesh,
            typeName: newTypeName,
            rotationY: 0 // Start with 0 rotation for the group
        });

        const currentCount = objectCountByType.get(newTypeName) || 0;
        objectCountByType.set(newTypeName, currentCount + 1);
    }

        let gradioClient = null;

        async function initializeGradioClient() {
            try {
                gradioClient = await Client.connect("jbilcke-hf/text-to-3d");
            } catch (error) {
                console.error("Failed to connect to Gradio:", error);
            }
        }

        async function generateTile(prompt) {
            if (!gradioClient) {
                await initializeGradioClient();
            }

            if (!gradioClient) {
                console.error("Gradio client not initialized");
                return;
            }

            if (prompt === lastPromptInputValue) {
                return;
            }

            lastPromptInputValue = prompt;

            try {
                isLoading = true;
                loadingOverlay.style.display = 'flex';

           
                const result = await gradioClient.predict("/generate", { 
                    prompt: `isometric videogame asset, ${prompt.trim()}`,
                });

                const modelUrl = result.data[0].url;
            

                //const modelUrl = "/models/eiffel_tower.glb";
                const tileTypeName = `custom_${Date.now()}`;
                await createGLBTileType(modelUrl, tileTypeName);
                
                // Instead of random placement, store the current tile type
                currentTileType = tileTypeName;
                
                // Optional: place one instance to show the new type
                replaceTileInstance(Math.floor(GRID_SIZE * GRID_SIZE / 2), tileTypeName);

            } catch (error) {
                console.error("Failed to generate/load terrain:", error);
            } finally {
                isLoading = false;
                loadingOverlay.style.display = 'none';
            }
        }


        // Handle window resize
        window.addEventListener('resize', () => {
            const aspect = window.innerWidth / window.innerHeight;
            const frustumSize = 40;
            camera.left = -frustumSize * aspect / 2;
            camera.right = frustumSize * aspect / 2;
            camera.top = frustumSize / 2;
            camera.bottom = -frustumSize / 2;
            camera.updateProjectionMatrix();
            renderer.setSize(window.innerWidth, window.innerHeight);
        });

        // Text input handling
        const promptInput = document.getElementById('prompt-input');

        // Event listeners
        promptInput.addEventListener('keypress', async (e) => {
            if (e.key === 'Enter') {
                await generateTile(promptInput.value);
            }
        });

        promptInput.addEventListener('blur', async () => {
            await generateTile(promptInput.value);
        });

        canvasH.addEventListener('click', createHeightMap);

        // Initialize and start
        createHeightMap();
        initializeTerrain();


        function onMouseDown(event) {
            if (event.button === 0) { // Left click
                mouseDownPosition.x = event.clientX;
                mouseDownPosition.y = event.clientY;
                isDragging = false;
            }
        }

            
        
        // Update raycasting to use current camera
        function onMouseMove(event) {
            if (event.buttons === 1) {
                const deltaX = Math.abs(event.clientX - mouseDownPosition.x);
                const deltaY = Math.abs(event.clientY - mouseDownPosition.y);
                if (deltaX > 5 || deltaY > 5) {
                    isDragging = true;
                }
            }

            mouse.x = (event.clientX / window.innerWidth) * 2 - 1;
            mouse.y = -(event.clientY / window.innerHeight) * 2 + 1;

            raycaster.setFromCamera(mouse, currentCamera);
            const intersects = raycaster.intersectObject(terrainInstances);

            if (intersects.length > 0) {
                const instanceId = intersects[0].instanceId;
                
                if (instanceId !== hoveredTileIndex) {
                    hoveredTileIndex = instanceId;
                    
                    const matrix = new THREE.Matrix4();
                    terrainInstances.getMatrixAt(instanceId, matrix);
                    const position = new THREE.Vector3();
                    matrix.decompose(position, new THREE.Quaternion(), new THREE.Vector3());
                    
                    highlightMesh.position.set(position.x, 0.4, position.z);
                    highlightMesh.visible = true;
                }
            } else {
                hoveredTileIndex = -1;
                highlightMesh.visible = false;
            }
        }

        function onMouseUp(event) {
            if (event.button === 0) { // Left click
                if (!isDragging && hoveredTileIndex !== -1 && currentTileType) {
                    replaceTileInstance(hoveredTileIndex, currentTileType);
                }
            }
        }

        function onClick(event) {
            if (event.button === 2) { // Right click
                if (hoveredTileIndex !== -1 && objectInstances.has(hoveredTileIndex)) {
                    const existing = objectInstances.get(hoveredTileIndex);
                    const typeName = existing.typeName;
                    if (typeName) {
                        const currentCount = objectCountByType.get(typeName) || 0;
                        if (currentCount > 1) {
                            objectCountByType.set(typeName, currentCount - 1);
                        } else {
                            objectCountByType.delete(typeName);
                        }
                    }
                    scene.remove(existing.rotationGroup);
                    objectInstances.delete(hoveredTileIndex);
                }
            }
        }


        // Add camera toggle button handler
        const cameraToggle = document.getElementById('camera-toggle');
        cameraToggle.addEventListener('click', () => {
            isOrthographic = !isOrthographic;
            currentCamera = isOrthographic ? orthoCamera : perspCamera;
            
            // Update controls
            controls.dispose();
            controls = new OrbitControls(currentCamera, renderer.domElement);
            controls.enableDamping = true;
            
            // Sync camera positions
            const oldPos = isOrthographic ? perspCamera.position : orthoCamera.position;
            currentCamera.position.copy(oldPos);
            currentCamera.lookAt(controls.target);
            
            // Update projection if needed
            if (!isOrthographic) {
                perspCamera.aspect = window.innerWidth / window.innerHeight;
                perspCamera.updateProjectionMatrix();
            }
        });

        // Update window resize handler
        window.addEventListener('resize', () => {
            const aspect = window.innerWidth / window.innerHeight;
            if (isOrthographic) {
                const frustumSize = 40;
                orthoCamera.left = -frustumSize * aspect / 2;
                orthoCamera.right = frustumSize * aspect / 2;
                orthoCamera.top = frustumSize / 2;
                orthoCamera.bottom = -frustumSize / 2;
                orthoCamera.updateProjectionMatrix();
            } else {
                perspCamera.aspect = aspect;
                perspCamera.updateProjectionMatrix();
            }
            renderer.setSize(window.innerWidth, window.innerHeight);
        });

        renderer.domElement.removeEventListener('click', onClick);
        renderer.domElement.addEventListener('mousedown', onMouseDown);
        renderer.domElement.addEventListener('mousemove', onMouseMove);
        renderer.domElement.addEventListener('mouseup', onMouseUp);
        renderer.domElement.addEventListener('contextmenu', (event) => {
            event.preventDefault();
            onClick(event);
        });


        function animate() {
            requestAnimationFrame(animate);
            controls.update();
            renderer.render(scene, currentCamera);
        }

        animate();
    </script>
</body>
</html>