|
|
""" |
|
|
Advanced 3D Reconstruction from Single Images with Responsible AI Features |
|
|
|
|
|
""" |
|
|
|
|
|
import gradio as gr |
|
|
import numpy as np |
|
|
import torch |
|
|
from PIL import Image |
|
|
from transformers import GLPNForDepthEstimation, GLPNImageProcessor |
|
|
import open3d as o3d |
|
|
import plotly.graph_objects as go |
|
|
import matplotlib.pyplot as plt |
|
|
import io |
|
|
import json |
|
|
import time |
|
|
from pathlib import Path |
|
|
import tempfile |
|
|
import zipfile |
|
|
import hashlib |
|
|
from datetime import datetime |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
RESPONSIBLE_AI_NOTICE = """ |
|
|
## ⚠️ Responsible Use Guidelines |
|
|
|
|
|
### Privacy & Consent |
|
|
- **Do not upload images containing identifiable people without their explicit consent** |
|
|
- **Do not use for surveillance, tracking, or monitoring individuals** |
|
|
- Facial features may be reconstructed in 3D - consider privacy implications |
|
|
- Remove metadata (EXIF) that may contain location or personal information |
|
|
|
|
|
### Ethical Use |
|
|
- This tool is for **educational, research, and creative purposes only** |
|
|
- **Prohibited uses:** |
|
|
- Creating deepfakes or misleading 3D content |
|
|
- Unauthorized documentation of private property |
|
|
- Circumventing security systems |
|
|
- Generating 3D models for harassment or stalking |
|
|
- Commercial use without proper rights to source images |
|
|
|
|
|
### Limitations & Bias |
|
|
- Models trained primarily on indoor Western architecture |
|
|
- May perform poorly on non-Western architectural styles |
|
|
- Scale is relative, not absolute - not suitable for precision measurements |
|
|
- Single viewpoint limitations - occluded areas are inferred, not captured |
|
|
|
|
|
### Data Usage |
|
|
- Images are processed locally during your session |
|
|
- No images are stored or transmitted to external servers |
|
|
- Processing logs contain only technical metrics, no image content |
|
|
- You retain all rights to your uploaded images and generated 3D models |
|
|
|
|
|
|
|
|
**By using this tool, you agree to these responsible use guidelines.** |
|
|
""" |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def check_image_safety(image): |
|
|
"""Basic safety checks for uploaded images""" |
|
|
warnings = [] |
|
|
|
|
|
width, height = image.size |
|
|
if width * height > 10_000_000: |
|
|
warnings.append("⚠️ Very large image - consider resizing to improve processing speed") |
|
|
|
|
|
aspect_ratio = max(width, height) / min(width, height) |
|
|
if aspect_ratio > 3: |
|
|
warnings.append("⚠️ Unusual aspect ratio detected - ensure image doesn't contain unintended content") |
|
|
|
|
|
try: |
|
|
exif = image.getexif() |
|
|
if exif: |
|
|
has_gps = any(k for k in exif.keys() if k in [34853, 0x8825]) |
|
|
if has_gps: |
|
|
warnings.append("⚠️ GPS location data detected in image - consider removing EXIF data for privacy") |
|
|
except: |
|
|
pass |
|
|
|
|
|
return True, "\n".join(warnings) if warnings else None |
|
|
|
|
|
def generate_session_id(): |
|
|
"""Generate anonymous session ID for logging""" |
|
|
return hashlib.sha256(str(datetime.now()).encode()).hexdigest()[:16] |
|
|
|
|
|
def content_policy_check(image): |
|
|
"""Check if image content violates usage policies""" |
|
|
width, height = image.size |
|
|
|
|
|
if width < 100 or height < 100: |
|
|
return False, "Image too small - minimum 100x100 pixels required for meaningful reconstruction" |
|
|
|
|
|
return True, None |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
print("Loading GLPN model (lightweight)...") |
|
|
try: |
|
|
glpn_processor = GLPNImageProcessor.from_pretrained("vinvino02/glpn-nyu") |
|
|
glpn_model = GLPNForDepthEstimation.from_pretrained("vinvino02/glpn-nyu") |
|
|
print("✓ GLPN model loaded successfully!") |
|
|
except Exception as e: |
|
|
print(f"Error loading model: {e}") |
|
|
glpn_processor = None |
|
|
glpn_model = None |
|
|
|
|
|
|
|
|
dpt_model = None |
|
|
dpt_processor = None |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def process_image(image, model_choice="GLPN (Recommended)", visualization_type="mesh"): |
|
|
"""Optimized processing pipeline""" |
|
|
|
|
|
def _generate_quality_assessment(metrics): |
|
|
assessment = [] |
|
|
outlier_pct = (metrics['outliers_removed'] / metrics['initial_points']) * 100 |
|
|
|
|
|
if outlier_pct < 5: |
|
|
assessment.append("Very clean depth estimation") |
|
|
elif outlier_pct < 15: |
|
|
assessment.append("Good depth quality") |
|
|
else: |
|
|
assessment.append("High noise in depth estimation") |
|
|
|
|
|
if metrics['is_edge_manifold'] and metrics['is_vertex_manifold']: |
|
|
assessment.append("Excellent topology") |
|
|
elif metrics['is_vertex_manifold']: |
|
|
assessment.append("Good local topology") |
|
|
else: |
|
|
assessment.append("Topology issues present") |
|
|
|
|
|
if metrics['is_watertight']: |
|
|
assessment.append("Watertight mesh - ready for 3D printing!") |
|
|
else: |
|
|
assessment.append("Not watertight - needs repair for 3D printing") |
|
|
|
|
|
return "\n".join(f"- {item}" for item in assessment) |
|
|
|
|
|
if glpn_model is None: |
|
|
return None, None, None, "❌ Model failed to load. Please refresh the page.", None |
|
|
|
|
|
try: |
|
|
print("Starting reconstruction...") |
|
|
|
|
|
|
|
|
new_height = 480 if image.height > 480 else image.height |
|
|
new_height -= (new_height % 32) |
|
|
new_width = int(new_height * image.width / image.height) |
|
|
diff = new_width % 32 |
|
|
new_width = new_width - diff if diff < 16 else new_width + (32 - diff) |
|
|
new_size = (new_width, new_height) |
|
|
image = image.resize(new_size, Image.LANCZOS) |
|
|
|
|
|
|
|
|
if model_choice == "GLPN (Recommended)": |
|
|
processor = glpn_processor |
|
|
model = glpn_model |
|
|
else: |
|
|
global dpt_model, dpt_processor |
|
|
if dpt_model is None: |
|
|
print("Loading DPT model (first time only)...") |
|
|
from transformers import DPTForDepthEstimation, DPTImageProcessor |
|
|
dpt_processor = DPTImageProcessor.from_pretrained("Intel/dpt-large") |
|
|
dpt_model = DPTForDepthEstimation.from_pretrained("Intel/dpt-large") |
|
|
print("✓ DPT model loaded!") |
|
|
processor = dpt_processor |
|
|
model = dpt_model |
|
|
|
|
|
inputs = processor(images=image, return_tensors="pt") |
|
|
|
|
|
start_time = time.time() |
|
|
with torch.no_grad(): |
|
|
outputs = model(**inputs) |
|
|
predicted_depth = outputs.predicted_depth |
|
|
depth_time = time.time() - start_time |
|
|
|
|
|
|
|
|
pad = 16 |
|
|
output = predicted_depth.squeeze().cpu().numpy() * 1000.0 |
|
|
output = output[pad:-pad, pad:-pad] |
|
|
image_cropped = image.crop((pad, pad, image.width - pad, image.height - pad)) |
|
|
|
|
|
depth_height, depth_width = output.shape |
|
|
img_width, img_height = image_cropped.size |
|
|
|
|
|
if depth_height != img_height or depth_width != img_width: |
|
|
from scipy import ndimage |
|
|
zoom_factors = (img_height / depth_height, img_width / depth_width) |
|
|
output = ndimage.zoom(output, zoom_factors, order=1) |
|
|
|
|
|
image = image_cropped |
|
|
|
|
|
|
|
|
fig, ax = plt.subplots(1, 2, figsize=(14, 7)) |
|
|
ax[0].imshow(image) |
|
|
ax[0].set_title('Original Image', fontsize=14, fontweight='bold') |
|
|
ax[0].axis('off') |
|
|
|
|
|
im = ax[1].imshow(output, cmap='plasma') |
|
|
ax[1].set_title('Estimated Depth Map', fontsize=14, fontweight='bold') |
|
|
ax[1].axis('off') |
|
|
plt.colorbar(im, ax=ax[1], fraction=0.046, pad=0.04) |
|
|
plt.tight_layout() |
|
|
|
|
|
buf = io.BytesIO() |
|
|
plt.savefig(buf, format='png', dpi=150, bbox_inches='tight') |
|
|
buf.seek(0) |
|
|
depth_viz = Image.open(buf) |
|
|
plt.close() |
|
|
|
|
|
|
|
|
width, height = image.size |
|
|
|
|
|
if output.shape != (height, width): |
|
|
from scipy import ndimage |
|
|
zoom_factors = (height / output.shape[0], width / output.shape[1]) |
|
|
output = ndimage.zoom(output, zoom_factors, order=1) |
|
|
|
|
|
depth_image = (output * 255 / np.max(output)).astype(np.uint8) |
|
|
image_array = np.array(image) |
|
|
|
|
|
depth_o3d = o3d.geometry.Image(depth_image) |
|
|
image_o3d = o3d.geometry.Image(image_array) |
|
|
rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth( |
|
|
image_o3d, depth_o3d, convert_rgb_to_intensity=False |
|
|
) |
|
|
|
|
|
camera_intrinsic = o3d.camera.PinholeCameraIntrinsic() |
|
|
camera_intrinsic.set_intrinsics(width, height, 500, 500, width/2, height/2) |
|
|
|
|
|
pcd = o3d.geometry.PointCloud.create_from_rgbd_image(rgbd_image, camera_intrinsic) |
|
|
initial_points = len(pcd.points) |
|
|
|
|
|
|
|
|
cl, ind = pcd.remove_statistical_outlier(nb_neighbors=20, std_ratio=2.0) |
|
|
pcd = pcd.select_by_index(ind) |
|
|
outliers_removed = initial_points - len(pcd.points) |
|
|
|
|
|
|
|
|
pcd.estimate_normals() |
|
|
pcd.orient_normals_to_align_with_direction() |
|
|
|
|
|
|
|
|
mesh_start = time.time() |
|
|
mesh = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson( |
|
|
pcd, depth=9, n_threads=1 |
|
|
)[0] |
|
|
|
|
|
|
|
|
pcd_tree = o3d.geometry.KDTreeFlann(pcd) |
|
|
mesh_colors = [] |
|
|
for vertex in mesh.vertices: |
|
|
[_, idx, _] = pcd_tree.search_knn_vector_3d(vertex, 1) |
|
|
mesh_colors.append(pcd.colors[idx[0]]) |
|
|
mesh.vertex_colors = o3d.utility.Vector3dVector(np.array(mesh_colors)) |
|
|
|
|
|
rotation = mesh.get_rotation_matrix_from_xyz((np.pi, 0, 0)) |
|
|
mesh.rotate(rotation, center=(0, 0, 0)) |
|
|
mesh_time = time.time() - mesh_start |
|
|
|
|
|
|
|
|
mesh.compute_vertex_normals() |
|
|
|
|
|
metrics = { |
|
|
'model_used': model_choice, |
|
|
'depth_estimation_time': f"{depth_time:.2f}s", |
|
|
'mesh_reconstruction_time': f"{mesh_time:.2f}s", |
|
|
'total_time': f"{depth_time + mesh_time:.2f}s", |
|
|
'initial_points': initial_points, |
|
|
'outliers_removed': outliers_removed, |
|
|
'final_points': len(pcd.points), |
|
|
'vertices': len(mesh.vertices), |
|
|
'triangles': len(mesh.triangles), |
|
|
'is_edge_manifold': mesh.is_edge_manifold(), |
|
|
'is_vertex_manifold': mesh.is_vertex_manifold(), |
|
|
'is_watertight': mesh.is_watertight(), |
|
|
} |
|
|
|
|
|
|
|
|
try: |
|
|
surface_area = mesh.get_surface_area() |
|
|
if surface_area > 0: |
|
|
metrics['surface_area'] = float(surface_area) |
|
|
else: |
|
|
vertices = np.asarray(mesh.vertices) |
|
|
triangles = np.asarray(mesh.triangles) |
|
|
v0 = vertices[triangles[:, 0]] |
|
|
v1 = vertices[triangles[:, 1]] |
|
|
v2 = vertices[triangles[:, 2]] |
|
|
cross = np.cross(v1 - v0, v2 - v0) |
|
|
areas = 0.5 * np.linalg.norm(cross, axis=1) |
|
|
metrics['surface_area'] = float(np.sum(areas)) |
|
|
except: |
|
|
metrics['surface_area'] = "Unable to compute" |
|
|
|
|
|
|
|
|
try: |
|
|
if mesh.is_watertight(): |
|
|
metrics['volume'] = float(mesh.get_volume()) |
|
|
else: |
|
|
metrics['volume'] = None |
|
|
except: |
|
|
metrics['volume'] = None |
|
|
|
|
|
|
|
|
points = np.asarray(pcd.points) |
|
|
colors = np.asarray(pcd.colors) |
|
|
|
|
|
if visualization_type == "point_cloud": |
|
|
scatter = go.Scatter3d( |
|
|
x=points[:, 0], y=points[:, 1], z=points[:, 2], |
|
|
mode='markers', |
|
|
marker=dict( |
|
|
size=2, |
|
|
color=['rgb({},{},{})'.format(int(r*255), int(g*255), int(b*255)) |
|
|
for r, g, b in colors], |
|
|
), |
|
|
name='Point Cloud' |
|
|
) |
|
|
|
|
|
plotly_fig = go.Figure(data=[scatter]) |
|
|
plotly_fig.update_layout( |
|
|
scene=dict( |
|
|
xaxis=dict(visible=False), |
|
|
yaxis=dict(visible=False), |
|
|
zaxis=dict(visible=False), |
|
|
aspectmode='data' |
|
|
), |
|
|
height=700, |
|
|
title="Point Cloud" |
|
|
) |
|
|
else: |
|
|
vertices = np.asarray(mesh.vertices) |
|
|
triangles = np.asarray(mesh.triangles) |
|
|
|
|
|
if mesh.has_vertex_colors(): |
|
|
vertex_colors = np.asarray(mesh.vertex_colors) |
|
|
colors_rgb = ['rgb({},{},{})'.format(int(r*255), int(g*255), int(b*255)) |
|
|
for r, g, b in vertex_colors] |
|
|
|
|
|
mesh_trace = go.Mesh3d( |
|
|
x=vertices[:, 0], y=vertices[:, 1], z=vertices[:, 2], |
|
|
i=triangles[:, 0], j=triangles[:, 1], k=triangles[:, 2], |
|
|
vertexcolor=colors_rgb, |
|
|
opacity=0.95 |
|
|
) |
|
|
else: |
|
|
mesh_trace = go.Mesh3d( |
|
|
x=vertices[:, 0], y=vertices[:, 1], z=vertices[:, 2], |
|
|
i=triangles[:, 0], j=triangles[:, 1], k=triangles[:, 2], |
|
|
color='lightblue', |
|
|
opacity=0.9 |
|
|
) |
|
|
|
|
|
plotly_fig = go.Figure(data=[mesh_trace]) |
|
|
plotly_fig.update_layout( |
|
|
scene=dict( |
|
|
xaxis=dict(visible=False), |
|
|
yaxis=dict(visible=False), |
|
|
zaxis=dict(visible=False), |
|
|
aspectmode='data' |
|
|
), |
|
|
height=700, |
|
|
title="3D Mesh" |
|
|
) |
|
|
|
|
|
|
|
|
temp_dir = tempfile.mkdtemp() |
|
|
|
|
|
pcd_path = Path(temp_dir) / "point_cloud.ply" |
|
|
o3d.io.write_point_cloud(str(pcd_path), pcd) |
|
|
|
|
|
mesh_path = Path(temp_dir) / "mesh.ply" |
|
|
o3d.io.write_triangle_mesh(str(mesh_path), mesh) |
|
|
|
|
|
mesh_obj_path = Path(temp_dir) / "mesh.obj" |
|
|
o3d.io.write_triangle_mesh(str(mesh_obj_path), mesh) |
|
|
|
|
|
mesh_stl_path = Path(temp_dir) / "mesh.stl" |
|
|
o3d.io.write_triangle_mesh(str(mesh_stl_path), mesh) |
|
|
|
|
|
metrics_path = Path(temp_dir) / "metrics.json" |
|
|
with open(metrics_path, 'w') as f: |
|
|
json.dump(metrics, f, indent=2, default=str) |
|
|
|
|
|
zip_path = Path(temp_dir) / "reconstruction_complete.zip" |
|
|
with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf: |
|
|
zipf.write(pcd_path, pcd_path.name) |
|
|
zipf.write(mesh_path, mesh_path.name) |
|
|
zipf.write(mesh_obj_path, mesh_obj_path.name) |
|
|
zipf.write(mesh_stl_path, mesh_stl_path.name) |
|
|
zipf.write(metrics_path, metrics_path.name) |
|
|
|
|
|
assessment = _generate_quality_assessment(metrics) |
|
|
|
|
|
report = f""" |
|
|
## Reconstruction Complete! |
|
|
|
|
|
### Performance |
|
|
- **Processing Time**: {metrics['total_time']} |
|
|
- **Points**: {metrics['final_points']:,} |
|
|
- **Triangles**: {metrics['triangles']:,} |
|
|
|
|
|
### Quality |
|
|
- **Topology**: {'Good' if metrics['is_vertex_manifold'] else 'Issues'} |
|
|
- **Watertight**: {'Yes' if metrics['is_watertight'] else 'No'} |
|
|
|
|
|
### Assessment |
|
|
{assessment} |
|
|
|
|
|
**Download the complete package below!** |
|
|
""" |
|
|
|
|
|
return depth_viz, plotly_fig, str(zip_path), report, json.dumps(metrics, indent=2, default=str) |
|
|
|
|
|
except Exception as e: |
|
|
import traceback |
|
|
return None, None, None, f"Error: {str(e)}\n\n{traceback.format_exc()}", None |
|
|
|
|
|
def process_image_with_safeguards(image, model_choice="GLPN (Recommended)", visualization_type="mesh", consent_given=False): |
|
|
"""Main processing with safeguards""" |
|
|
session_id = generate_session_id() |
|
|
|
|
|
if not consent_given: |
|
|
return None, None, None, "**You must agree to the Responsible Use Guidelines first.**", None |
|
|
|
|
|
if image is None: |
|
|
return None, None, None, "Please upload an image first.", None |
|
|
|
|
|
is_safe, safety_warning = check_image_safety(image) |
|
|
passes_policy, policy_message = content_policy_check(image) |
|
|
|
|
|
if not passes_policy: |
|
|
return None, None, None, f"{policy_message}", None |
|
|
|
|
|
try: |
|
|
result = process_image(image, model_choice, visualization_type) |
|
|
depth_viz, plotly_fig, zip_path, report, json_metrics = result |
|
|
|
|
|
if safety_warning: |
|
|
report = f"**Privacy Notice:**\n{safety_warning}\n\n{report}" |
|
|
|
|
|
metrics = json.loads(json_metrics) |
|
|
metrics['responsible_ai'] = { |
|
|
'session_id': session_id, |
|
|
'timestamp': datetime.now().isoformat(), |
|
|
'consent_given': True |
|
|
} |
|
|
|
|
|
return depth_viz, plotly_fig, zip_path, report, json.dumps(metrics, indent=2) |
|
|
|
|
|
except Exception as e: |
|
|
return None, None, None, f"Error: {str(e)}", None |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
with gr.Blocks(title="Responsible AI 3D Reconstruction", theme=gr.themes.Soft()) as demo: |
|
|
|
|
|
gr.Markdown(""" |
|
|
# 🏗️ 3D Reconstruction from Single Images |
|
|
|
|
|
|
|
|
Transform 2D photographs into 3D spatial models |
|
|
|
|
|
<div style="background-color: #fff3cd; border: 2px solid #ffc107; padding: 15px; border-radius: 5px; margin: 10px 0;"> |
|
|
<h3 style="color: #856404; margin-top: 0;">⚠️ Responsible Use Required</h3> |
|
|
<p style="color: #856404; margin-bottom: 0;">This tool must be used ethically and legally. Review the guidelines in the <b>first tab</b>.</p> |
|
|
</div> |
|
|
""") |
|
|
|
|
|
with gr.Tabs(): |
|
|
|
|
|
with gr.Tab("⚠️ Responsible Use (READ FIRST)"): |
|
|
gr.Markdown(RESPONSIBLE_AI_NOTICE) |
|
|
gr.Markdown(""" |
|
|
### Known Limitations & Biases |
|
|
- Trained primarily on Western indoor architecture |
|
|
- May underperform on non-Western styles |
|
|
- Scale is relative, not absolute |
|
|
- Single viewpoint captures only visible surfaces |
|
|
""") |
|
|
|
|
|
with gr.Tab("Reconstruction"): |
|
|
consent_checkbox = gr.Checkbox( |
|
|
label="**I have read and agree to the Responsible Use Guidelines**", |
|
|
value=False |
|
|
) |
|
|
|
|
|
with gr.Row(): |
|
|
with gr.Column(scale=1): |
|
|
input_image = gr.Image( |
|
|
type="pil", |
|
|
label="Upload Image", |
|
|
sources=["upload", "clipboard"] |
|
|
) |
|
|
|
|
|
model_choice = gr.Radio( |
|
|
choices=["GLPN (Recommended)", "DPT (High Quality)"], |
|
|
value="GLPN (Recommended)", |
|
|
label="Depth Estimation Model" |
|
|
) |
|
|
|
|
|
visualization_type = gr.Radio( |
|
|
choices=["mesh", "point_cloud"], |
|
|
value="mesh", |
|
|
label="Visualization Type" |
|
|
) |
|
|
|
|
|
reconstruct_btn = gr.Button("Start Reconstruction", variant="primary", size="lg") |
|
|
|
|
|
with gr.Column(scale=2): |
|
|
depth_output = gr.Image(label="Depth Map") |
|
|
viewer_3d = gr.Plot(label="Interactive 3D Viewer") |
|
|
|
|
|
with gr.Row(): |
|
|
with gr.Column(): |
|
|
metrics_output = gr.Markdown(label="Report") |
|
|
with gr.Column(): |
|
|
json_output = gr.Textbox(label="Metrics (JSON)", lines=8) |
|
|
|
|
|
download_output = gr.File(label="Download Package (ZIP)") |
|
|
|
|
|
reconstruct_btn.click( |
|
|
fn=process_image_with_safeguards, |
|
|
inputs=[input_image, model_choice, visualization_type, consent_checkbox], |
|
|
outputs=[depth_output, viewer_3d, download_output, metrics_output, json_output] |
|
|
) |
|
|
|
|
|
with gr.Tab("Theory & Background"): |
|
|
gr.Markdown(""" |
|
|
## About This Tool |
|
|
|
|
|
This application demonstrates how artificial intelligence can convert single 2D photographs |
|
|
into interactive 3D models automatically. |
|
|
|
|
|
### What Makes This Special |
|
|
|
|
|
**Traditional Approach:** |
|
|
- Need special equipment (3D scanner, multiple cameras) |
|
|
- Requires technical expertise |
|
|
- Time-consuming process |
|
|
- Expensive |
|
|
|
|
|
**This AI Approach:** |
|
|
- Works with any single photograph |
|
|
- No special equipment needed |
|
|
- Automatic processing |
|
|
- Free and accessible |
|
|
|
|
|
|
|
|
|
|
|
## The Technology |
|
|
|
|
|
### AI Model Used: GLPN |
|
|
|
|
|
**GLPN (Global-Local Path Networks)** |
|
|
- Paper: Kim et al., CVPR 2022 |
|
|
- Optimized for: Indoor/outdoor architectural scenes |
|
|
- Training: NYU Depth V2 (urban indoor environments) |
|
|
- Best for: Building interiors, street-level views |
|
|
- Speed: Fast (~0.3-2.5s) |
|
|
|
|
|
### How It Works (Simplified) |
|
|
|
|
|
1. **AI analyzes photo** → Recognizes objects, patterns, perspective |
|
|
2. **Estimates distance** → Figures out what's close, what's far |
|
|
3. **Creates 3D points** → Places colored dots in 3D space |
|
|
4. **Builds surface** → Connects dots into smooth shape |
|
|
|
|
|
### Spatial Data Pipeline |
|
|
|
|
|
**1. Monocular Depth Estimation** |
|
|
- Challenge: Extracting 3D spatial information from 2D photographs |
|
|
- Application: Similar to photogrammetry but from single images |
|
|
- Output: Relative depth maps for spatial analysis |
|
|
|
|
|
**2. Point Cloud Generation** |
|
|
- Creates 3D coordinate system (X, Y, Z) from pixels |
|
|
- Each point: Spatial location + RGB color information |
|
|
- Compatible with: GIS software, CAD tools, spatial databases |
|
|
|
|
|
**3. 3D Mesh Generation** |
|
|
- Creates continuous surface from discrete points |
|
|
- Similar to: Digital terrain models (DTMs) for buildings |
|
|
- Output formats: Compatible with ArcGIS, QGIS, SketchUp |
|
|
|
|
|
### Quality Metrics Explained |
|
|
|
|
|
- **Point Cloud Density**: Higher points = better spatial resolution |
|
|
- **Geometric Accuracy**: Manifold checks ensure valid topology |
|
|
- **Surface Continuity**: Watertight meshes = complete volume calculations |
|
|
- **Data Fidelity**: Triangle count indicates level of detail |
|
|
|
|
|
### Limitations for Geographic Applications |
|
|
|
|
|
1. **Scale Ambiguity**: Requires ground control points for absolute measurements |
|
|
2. **Single Viewpoint**: Cannot capture occluded facades or hidden spaces |
|
|
3. **No Georeferencing**: Outputs in local coordinates, not global (lat/lon) |
|
|
4. **Weather Dependent**: Best results with clear, well-lit conditions |
|
|
|
|
|
### Comparison with Traditional Methods |
|
|
|
|
|
**vs. Terrestrial Laser Scanning (TLS):** |
|
|
- Much cheaper, faster, more accessible |
|
|
- Lower accuracy, no absolute scale |
|
|
|
|
|
**vs. Photogrammetry (Structure-from-Motion):** |
|
|
- Works with single image, faster processing |
|
|
- Less accurate, cannot resolve scale |
|
|
|
|
|
**vs. LiDAR:** |
|
|
- Much lower cost, consumer cameras sufficient |
|
|
- Lower precision, no absolute measurements |
|
|
|
|
|
|
|
|
|
|
|
## Reconstruction Pipeline (10 Steps) |
|
|
|
|
|
1. **Image Preprocessing**: Resize to model requirements |
|
|
2. **Depth Estimation**: Neural network inference |
|
|
3. **Depth Visualization**: Create comparison images |
|
|
4. **Point Cloud Generation**: Back-project using camera model |
|
|
5. **Outlier Removal**: Statistical filtering |
|
|
6. **Normal Estimation**: Surface orientation calculation |
|
|
7. **Mesh Reconstruction**: Poisson surface reconstruction |
|
|
8. **Quality Metrics**: Compute geometric measures |
|
|
9. **3D Visualization**: Create interactive viewer |
|
|
10. **File Export**: Generate multiple formats |
|
|
|
|
|
### Key References |
|
|
|
|
|
1. **Kim, D., et al. (2022)**. "Global-Local Path Networks for Monocular Depth Estimation |
|
|
with Vertical CutDepth." *CVPR 2022* |
|
|
2. **Kazhdan, M., et al. (2006)**. "Poisson Surface Reconstruction." |
|
|
*Eurographics Symposium on Geometry Processing* |
|
|
""") |
|
|
|
|
|
with gr.Tab("Usage Guide"): |
|
|
gr.Markdown(""" |
|
|
## How to Use This Application |
|
|
|
|
|
### Step 1: Read Responsible Use Guidelines |
|
|
- **REQUIRED**: Review the "Responsible Use" tab first |
|
|
- Understand privacy implications |
|
|
- Acknowledge model limitations and biases |
|
|
- Ensure you have rights to use source images |
|
|
|
|
|
### Step 2: Prepare Your Image |
|
|
|
|
|
**Best Practices:** |
|
|
- Remove EXIF metadata (GPS, timestamps) for privacy |
|
|
- Ensure you have consent if image contains people |
|
|
- Use well-lit, clear photographs |
|
|
- Recommended resolution: 512-1024 pixels |
|
|
- Indoor scenes work best |
|
|
|
|
|
**Privacy Checklist:** |
|
|
- [ ] No identifiable people (or consent obtained) |
|
|
- [ ] No sensitive/private locations |
|
|
- [ ] EXIF data removed |
|
|
- [ ] You own rights to the image |
|
|
|
|
|
### Step 3: Upload Image |
|
|
- Click "Upload Image" area |
|
|
- Select JPG, PNG, or BMP file |
|
|
- **Note:** Webcam option removed for privacy protection |
|
|
- You can also paste from clipboard |
|
|
|
|
|
### Step 4: Check Consent Box |
|
|
- Check "I have read and agree to Responsible Use Guidelines" |
|
|
- This confirms you've reviewed ethical guidelines |
|
|
- Processing won't start without consent |
|
|
|
|
|
### Step 5: Choose Visualization |
|
|
- **Mesh**: Solid 3D surface (recommended) |
|
|
- **Point Cloud**: Individual 3D points with colors |
|
|
|
|
|
### Step 6: Start Reconstruction |
|
|
- Click "Start Reconstruction" |
|
|
- Processing takes 10-60 seconds |
|
|
- All processing is local (no cloud upload) |
|
|
|
|
|
### Step 7: Explore Results |
|
|
|
|
|
**Depth Map:** |
|
|
- Yellow/Red = Farther objects |
|
|
- Purple/Blue = Closer objects |
|
|
- Shows AI's depth understanding |
|
|
|
|
|
**3D Viewer:** |
|
|
- Rotate: Click and drag |
|
|
- Zoom: Scroll wheel |
|
|
- Pan: Right-click and drag |
|
|
- Reset: Double-click |
|
|
|
|
|
**Metrics Report:** |
|
|
- Processing performance |
|
|
- Quality indicators |
|
|
- Topology validation |
|
|
|
|
|
### Step 8: Download Files |
|
|
- ZIP package contains: |
|
|
- Point cloud (PLY) |
|
|
- Mesh (PLY, OBJ, STL) |
|
|
- Quality metrics (JSON) |
|
|
- All files include responsible AI metadata |
|
|
|
|
|
|
|
|
|
|
|
## Viewing Downloaded 3D Files |
|
|
|
|
|
### Free Software Options: |
|
|
|
|
|
**MeshLab** (Recommended for beginners) |
|
|
- Download: https://www.meshlab.net/ |
|
|
- Open PLY, OBJ, STL files |
|
|
- Great for viewing and basic editing |
|
|
|
|
|
**Blender** (For advanced users) |
|
|
- Download: https://www.blender.org/ |
|
|
- Import → Wavefront (.obj) or PLY |
|
|
- Full 3D modeling and rendering capabilities |
|
|
|
|
|
**CloudCompare** (For point clouds) |
|
|
- Download: https://www.cloudcompare.org/ |
|
|
- Best for analyzing point cloud data |
|
|
- Measurement and analysis tools |
|
|
|
|
|
**Online Viewers** (No installation) |
|
|
- https://3dviewer.net/ |
|
|
- https://www.creators3d.com/online-viewer |
|
|
- Just drag and drop your OBJ/PLY file |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
## Tips for Best Results |
|
|
|
|
|
### DO: |
|
|
- Use well-lit images |
|
|
- Include depth cues (corners, edges) |
|
|
- Indoor scenes work best |
|
|
- Medium resolution (512-1024px) |
|
|
- Remove personal metadata |
|
|
- Obtain consent for people in images |
|
|
|
|
|
### AVOID: |
|
|
- Motion blur or low resolution |
|
|
- Reflective surfaces (mirrors, glass) |
|
|
- Images without consent |
|
|
- Private property without permission |
|
|
- Surveillance or monitoring purposes |
|
|
- Heavy shadows or darkness |
|
|
|
|
|
|
|
|
## Understanding the Metrics |
|
|
|
|
|
### Point Cloud Statistics: |
|
|
- **Initial Points**: Raw points generated from depth |
|
|
- **Outliers Removed**: Noisy points filtered out (typically 5-15%) |
|
|
- **Final Points**: Clean points used for mesh generation |
|
|
|
|
|
### Mesh Quality Indicators: |
|
|
- ** Edge Manifold**: Each edge connects exactly 2 faces (good topology) |
|
|
- ** Vertex Manifold**: Clean vertex connections |
|
|
- ** Watertight**: No holes, ready for 3D printing |
|
|
- ** Marks**: Indicate potential issues (still usable, may need repair) |
|
|
|
|
|
### Processing Times: |
|
|
- **Depth Estimation**: 0.3-2.5s (GLPN model) |
|
|
- **Mesh Reconstruction**: 2-10s (depends on point cloud size) |
|
|
- **Total Time**: Usually 10-60 seconds |
|
|
|
|
|
--- |
|
|
|
|
|
## Troubleshooting |
|
|
|
|
|
**Problem: No output appears** |
|
|
- Check browser console for errors |
|
|
- Try refreshing the page |
|
|
- Try a smaller/simpler image first |
|
|
- Check that image uploaded successfully |
|
|
|
|
|
**Problem: Mesh has holes or artifacts** |
|
|
- This is normal for single-view reconstruction |
|
|
- Hidden surfaces cannot be reconstructed |
|
|
- Use mesh repair tools in MeshLab if needed |
|
|
|
|
|
**Problem: Colors look wrong on mesh** |
|
|
- Vertex color interpolation is approximate |
|
|
- This is expected behavior |
|
|
- Colors on point cloud are more accurate |
|
|
|
|
|
**Problem: Processing is very slow** |
|
|
- Use smaller images |
|
|
- This is normal on CPU (GPU is much faster) |
|
|
|
|
|
**Problem: "Not watertight" in metrics** |
|
|
- Common for complex scenes |
|
|
- Still usable for visualization |
|
|
- For 3D printing: use mesh repair in MeshLab |
|
|
""") |
|
|
|
|
|
with gr.Tab(" Ethics & Impact"): |
|
|
gr.Markdown(""" |
|
|
|
|
|
## Algorithmic Bias & Fairness |
|
|
|
|
|
### Training Data Representation |
|
|
|
|
|
**Geographic Bias:** |
|
|
- Heavy representation: North America, Europe |
|
|
- Underrepresented: Africa, South Asia, Pacific Islands |
|
|
- Impact: Lower accuracy for non-Western architecture |
|
|
|
|
|
**Architectural Style Bias:** |
|
|
- Well-represented: Modern interiors, Western buildings |
|
|
- Underrepresented: Traditional, vernacular, indigenous structures |
|
|
- Impact: May misinterpret non-standard spatial layouts |
|
|
|
|
|
**Socioeconomic Bias:** |
|
|
- Training data skewed toward middle/upper-class interiors |
|
|
- Limited representation of informal settlements |
|
|
- May not generalize well to all socioeconomic contexts |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
### Potential Harms |
|
|
|
|
|
** Privacy Violations:** |
|
|
- Unauthorized 3D reconstruction of private spaces |
|
|
- Creating models of individuals without consent |
|
|
- Surveillance and tracking applications |
|
|
|
|
|
** Misinformation:** |
|
|
- Generating fake 3D evidence |
|
|
- Manipulating spatial understanding |
|
|
- Creating misleading visualizations |
|
|
|
|
|
** Property Rights:** |
|
|
- Unauthorized documentation of copyrighted designs |
|
|
- Intellectual property theft |
|
|
- Commercial exploitation without permission |
|
|
|
|
|
### Harm Prevention |
|
|
|
|
|
1. **Mandatory consent**: Require user acknowledgment |
|
|
2. **Use case restriction**: Prohibit surveillance and deceptive uses |
|
|
3. **Privacy protection**: Disable webcam, encourage EXIF removal |
|
|
4. **Transparency**: Clear documentation of limitations |
|
|
|
|
|
|
|
|
|
|
|
## Accountability & Governance |
|
|
|
|
|
### User Responsibilities |
|
|
|
|
|
As a user, you are responsible for: |
|
|
- Ensuring lawful use of source images |
|
|
- Obtaining necessary consents and permissions |
|
|
- Respecting privacy and intellectual property |
|
|
- Using outputs ethically and transparently |
|
|
- Understanding and accounting for model biases |
|
|
|
|
|
### Developer Responsibilities |
|
|
|
|
|
This tool implements: |
|
|
- Clear responsible use guidelines |
|
|
- Privacy-protective design (no webcam, local processing) |
|
|
- Bias documentation and transparency |
|
|
- Prohibited use cases explicitly stated |
|
|
|
|
|
|
|
|
## Future Directions |
|
|
|
|
|
### Improving Fairness |
|
|
- Train on more diverse geographic datasets |
|
|
- Include underrepresented architectural styles |
|
|
- Develop bias mitigation techniques |
|
|
- Community-driven model evaluation |
|
|
|
|
|
### Enhancing Privacy |
|
|
- Face/person detection and redaction |
|
|
- Automatic EXIF stripping |
|
|
- Differential privacy techniques |
|
|
""") |
|
|
|
|
|
with gr.Tab(" Citation"): |
|
|
gr.Markdown(""" |
|
|
## Academic Citation |
|
|
|
|
|
### For GLPN Model: |
|
|
```bibtex |
|
|
@inproceedings{kim2022global, |
|
|
title={Global-Local Path Networks for Monocular Depth Estimation with Vertical CutDepth}, |
|
|
author={Kim, Doyeon and Ga, Woonghyun and Ahn, Pyungwhan and Joo, Donggyu and Chun, Sehwan and Kim, Junmo}, |
|
|
booktitle={CVPR}, |
|
|
year={2022} |
|
|
} |
|
|
``` |
|
|
|
|
|
### For Poisson Surface Reconstruction: |
|
|
```bibtex |
|
|
@inproceedings{kazhdan2006poisson, |
|
|
title={Poisson Surface Reconstruction}, |
|
|
author={Kazhdan, Michael and Bolitho, Matthew and Hoppe, Hugues}, |
|
|
booktitle={Symposium on Geometry Processing}, |
|
|
year={2006} |
|
|
} |
|
|
``` |
|
|
|
|
|
## Open Source Components |
|
|
|
|
|
This application is built with: |
|
|
|
|
|
- **Transformers** (Hugging Face): Model inference framework |
|
|
- **Open3D**: Point cloud and mesh processing |
|
|
- **PyTorch**: Deep learning framework |
|
|
- **Plotly**: Interactive 3D visualization |
|
|
- **Gradio**: Web interface framework |
|
|
- **NumPy** & **SciPy**: Numerical computing |
|
|
- **Matplotlib**: Data visualization |
|
|
- **Pillow (PIL)**: Image processing |
|
|
|
|
|
## Model Credits |
|
|
|
|
|
**GLPN Model:** |
|
|
- Developed by: KAIST (Korea Advanced Institute of Science and Technology) |
|
|
- Hosted by: Hugging Face (vinvino02/glpn-nyu) |
|
|
- License: Apache 2.0 |
|
|
|
|
|
## Responsible AI Features |
|
|
|
|
|
This implementation includes: |
|
|
- Privacy-protective design (no webcam option) |
|
|
- Mandatory consent acknowledgment |
|
|
- Bias documentation and transparency |
|
|
- Ethical use guidelines |
|
|
|
|
|
|
|
|
|
|
|
""") |
|
|
|
|
|
gr.Markdown(""" |
|
|
--- |
|
|
|
|
|
**Version:** 2.0 (Responsible AI Edition - Optimized) |
|
|
**Last Updated:** 2025 |
|
|
**License:** Educational and Research Use |
|
|
|
|
|
""") |
|
|
|
|
|
if __name__ == "__main__": |
|
|
print("="*60) |
|
|
print("RESPONSIBLE AI 3D RECONSTRUCTION") |
|
|
print("="*60) |
|
|
print("✓ Lightweight model (GLPN only)") |
|
|
print("✓ No webcam option") |
|
|
print("✓ Local processing") |
|
|
print("✓ Consent required") |
|
|
print("="*60) |
|
|
demo.launch(share=True) |
