Spaces:

johnrobinsn
/

MidasDepthEstimation

Running

App Files Files Community

johnrobinsn commited on Nov 2, 2022

Commit

831c6b2

•

1 Parent(s): a69b761

Initial Push

Browse files

Files changed (7) hide show

__pycache__/depth_viewer.cpython-39.pyc +0 -0
app.py +52 -0
depth_viewer.py +160 -0
examples/kitten.jpg +0 -0
examples/marsattacks.jpg +0 -0
examples/owl1.jpg +0 -0
requirements.txt +6 -0

__pycache__/depth_viewer.cpython-39.pyc ADDED Viewed

Binary file (5.31 kB). View file

app.py ADDED Viewed

	@@ -0,0 +1,52 @@

+import gradio as gr
+from transformers import DPTFeatureExtractor, DPTForDepthEstimation
+import torch
+import numpy as np
+from PIL import Image
+import open3d as o3d
+from pathlib import Path
+from depth_viewer import depthviewer2html
+feature_extractor = DPTFeatureExtractor.from_pretrained("Intel/dpt-large")
+model = DPTForDepthEstimation.from_pretrained("Intel/dpt-large")
+def process_image(image_path):
+    image_path = Path(image_path)
+    image = Image.open(image_path)
+    # prepare image for the model
+    encoding = feature_extractor(image, return_tensors="pt")
+    # forward pass
+    with torch.no_grad():
+        outputs = model(**encoding)
+        predicted_depth = outputs.predicted_depth
+    # interpolate to original size
+    prediction = torch.nn.functional.interpolate(
+        predicted_depth.unsqueeze(1),
+        size=image.size[::-1],
+        mode="bicubic",
+        align_corners=False,
+    ).squeeze()
+    output = prediction.cpu().numpy()
+    depth = (output * 255 / np.max(output)).astype('uint8')
+    h = depthviewer2html(image,depth)
+    return [h]
+title = "3d Visualization of Depth Maps Generated using MiDaS"
+description = "Improved 3D interactive depth viewer using Three.js embedded in a Gradio app. For more details see the <a href='https://colab.research.google.com/drive/1l2l8U7Vhq9RnvV2tHyfhrXKNuHfmb4IP?usp=sharing'>Colab Notebook.</a>"
+examples = [["examples/owl1.jpg"],['examples/marsattacks.jpg'],['examples/kitten.jpg']]
+iface = gr.Interface(fn=process_image,
+                     inputs=[gr.Image(type="filepath",label="Input Image")],
+                     outputs=[gr.HTML(label='Depth Viewer',elem_id='depth-viewer')],
+                     title=title,
+                     description=description,
+                     examples=examples,
+                     allow_flagging="never",
+                     cache_examples=False,
+                     css='#depth-viewer: {height:300px;}')
+iface.launch(debug=True, enable_queue=False)

depth_viewer.py ADDED Viewed

	@@ -0,0 +1,160 @@

+import cv2
+import base64
+import numpy as np
+_viewer_html = '''
+<html>
+<head>
+    <style>
+        body {
+            overflow: hidden;
+            margin: 0;
+        }
+    </style>
+    <script>
+        var image_url = "{{{image_url_marker}}}";
+        var depth_url = "{{{depth_url_marker}}}";
+        var blah;
+        function getImageData( image ) {
+            var canvas = document.createElement( 'canvas' );
+            canvas.width = image.width;
+            canvas.height = image.height;
+            var context = canvas.getContext( '2d' );
+            context.drawImage( image, 0, 0 );
+            return context.getImageData( 0, 0, image.width, image.height );
+        }
+        window.onload = (e) => {
+            var scene = new THREE.Scene();
+            scene.background = new THREE.Color( 0xffffff );
+            var camera = new THREE.PerspectiveCamera(60, window.innerWidth / window.innerHeight, 0.1, 1000);
+            camera.position.set(0, 0, 2);
+            var renderer = new THREE.WebGLRenderer({
+                antialias: true
+            });
+            renderer.setSize(window.innerWidth, window.innerHeight);
+            document.body.appendChild(renderer.domElement);
+            var controls = new THREE.OrbitControls(camera, renderer.domElement);
+            var reset = document.getElementById('reset')
+            reset.addEventListener('click', e => controls.reset())
+            function vertexShader() {
+                return document.getElementById("vshader").text
+            }
+            function fragmentShader() {
+                return document.getElementById("fshader").text
+            }
+            // The url for the image is passed in to the web application
+            //var p = new URLSearchParams(window.location.search);
+            //var image_url = 'https://tujot.com/thr/marsattacks.jpg';//p.get("image");
+            //var depth_url = '/depth?image='+image_url;
+            var texture = new THREE.TextureLoader().load(image_url, t => {
+                var w = t.image.width;
+                var h = t.image.height;
+                var max = Math.max(w, h);
+                var ar = w / h;
+                blah = getImageData(t.image);
+                console.log('texture:', getImageData(t.image).data)
+                var planeGeometry = new THREE.PlaneGeometry(w / max, h / max, w, h);
+                var depth = new THREE.TextureLoader().load(depth_url);
+                uniforms = {
+                    image: { type: "t", value: texture },
+                    depth: { type: "t", value: depth },
+                    ar: { type: 'f', value: ar }
+                }
+                let planeMaterial = new THREE.ShaderMaterial({
+                    uniforms: uniforms,
+                    fragmentShader: fragmentShader(),
+                    vertexShader: vertexShader(),
+                    side: THREE.DoubleSide
+                });
+                var points = new THREE.Points(planeGeometry, planeMaterial)
+                points.position.set(0, 0, 0)
+                scene.add(points)
+                render();
+            });
+            function render() {
+                requestAnimationFrame(render);
+                renderer.render(scene, camera);
+            }
+        }
+    </script>
+</head>
+<body>
+    <script src="https://threejs.org/build/three.min.js"></script>
+    <script src="https://threejs.org/examples/js/controls/OrbitControls.js"></script>
+    <script id="vshader" type="x-shader/x-vertex">
+        uniform sampler2D depth;
+        uniform float ar;
+        varying vec3 vUv;
+        vec3 pos;
+        void main() {
+          vUv = position;
+          pos = position;
+          pos.z = texture2D(depth,(vec2(vUv.x,vUv.y*ar)+0.5)).r;
+          float s = 2.0 - pos.z;
+          pos.x = pos.x * s;
+          pos.y = pos.y * s;
+          vec4 modelViewPosition = modelViewMatrix * vec4(pos, 1.0);
+          gl_Position = projectionMatrix * modelViewPosition;
+          gl_PointSize = 2.0;
+        }
+    </script>
+    <script id="fshader" type="x-shader/x-fragment">
+        uniform sampler2D image;
+        uniform float ar;
+        varying vec3 vUv;
+        void main() {
+          gl_FragColor = texture2D(image,(vec2(vUv.x,vUv.y*ar)+0.5));
+        }
+    </script>
+    <div style="position:absolute">
+        <button id="reset">Reset</button>
+    </div>
+</body>
+</html>
+'''
+image_url_marker = '{{{image_url_marker}}}'
+depth_url_marker = '{{{depth_url_marker}}}'
+def depthviewer2html(image,depth):
+    image_rgb = cv2.cvtColor(np.array(image), cv2.COLOR_BGR2RGB)
+    _, buffer = cv2.imencode('.jpg',image_rgb)
+    image_data_url = 'data:image/jpeg;base64,'+base64.b64encode(buffer).decode('utf-8')
+    _, buffer = cv2.imencode('.png',np.array(depth))
+    mask_data_url = 'data:image/png;base64,'+base64.b64encode(buffer).decode('utf-8')
+    vhtml = str(_viewer_html).replace(image_url_marker,image_data_url).replace(depth_url_marker,mask_data_url)
+    e = base64.b64encode(bytes(vhtml,'utf-8')).decode('utf-8')
+    url = f'data:text/html;base64,{e}'
+    h = f'<iframe src="{url}" height="600" width="100%"></iframe>'
+    return h

examples/kitten.jpg ADDED Viewed

examples/marsattacks.jpg ADDED Viewed

examples/owl1.jpg ADDED Viewed

requirements.txt ADDED Viewed

	@@ -0,0 +1,6 @@

+torch
+git+https://github.com/nielsrogge/transformers.git@add_dpt_redesign#egg=transformers
+numpy
+Pillow
+gradio==3.0b8
+opencv-python