Spaces:
Paused
Paused
| import numpy as np | |
| import trimesh | |
| from typing import Union, Dict, List, Tuple, Optional | |
| import tempfile | |
| from pathlib import Path | |
| class UniRigProcessor: | |
| """Automatic rigging for 3D models using simplified UniRig approach""" | |
| def __init__(self, device: str = "cuda"): | |
| self.device = device | |
| self.model = None | |
| # Rigging parameters | |
| self.bone_detection_threshold = 0.1 | |
| self.max_bones = 20 | |
| self.min_bones = 5 | |
| # Animation presets for monsters | |
| self.animation_presets = { | |
| 'idle': self._create_idle_animation, | |
| 'walk': self._create_walk_animation, | |
| 'attack': self._create_attack_animation, | |
| 'happy': self._create_happy_animation | |
| } | |
| def load_model(self): | |
| """Load rigging model (placeholder for actual implementation)""" | |
| # In production, this would load the actual UniRig model | |
| # For now, we'll use procedural rigging | |
| self.model = "procedural" | |
| def rig_mesh(self, | |
| mesh: Union[str, trimesh.Trimesh], | |
| mesh_type: str = "monster") -> Dict[str, any]: | |
| """Add rigging to a 3D mesh""" | |
| try: | |
| # Load mesh if path provided | |
| if isinstance(mesh, str): | |
| mesh = trimesh.load(mesh) | |
| # Ensure model is loaded | |
| if self.model is None: | |
| self.load_model() | |
| # Analyze mesh structure | |
| mesh_analysis = self._analyze_mesh(mesh) | |
| # Generate skeleton | |
| skeleton = self._generate_skeleton(mesh, mesh_analysis) | |
| # Compute bone weights | |
| weights = self._compute_bone_weights(mesh, skeleton) | |
| # Create rigged model | |
| rigged_model = { | |
| 'mesh': mesh, | |
| 'skeleton': skeleton, | |
| 'weights': weights, | |
| 'animations': self._create_default_animations(skeleton), | |
| 'metadata': { | |
| 'mesh_type': mesh_type, | |
| 'bone_count': len(skeleton['bones']), | |
| 'vertex_count': len(mesh.vertices) | |
| } | |
| } | |
| # Save rigged model | |
| output_path = self._save_rigged_model(rigged_model) | |
| return output_path | |
| except Exception as e: | |
| print(f"Rigging error: {e}") | |
| # Return original mesh if rigging fails | |
| return self._save_mesh_without_rigging(mesh) | |
| def _analyze_mesh(self, mesh: trimesh.Trimesh) -> Dict[str, any]: | |
| """Analyze mesh structure for rigging""" | |
| # Get mesh bounds and center | |
| bounds = mesh.bounds | |
| center = mesh.centroid | |
| # Analyze mesh topology | |
| analysis = { | |
| 'bounds': bounds, | |
| 'center': center, | |
| 'height': bounds[1][2] - bounds[0][2], | |
| 'width': bounds[1][0] - bounds[0][0], | |
| 'depth': bounds[1][1] - bounds[0][1], | |
| 'is_symmetric': self._check_symmetry(mesh), | |
| 'detected_limbs': self._detect_limbs(mesh), | |
| 'mesh_type': self._classify_mesh_type(mesh) | |
| } | |
| return analysis | |
| def _check_symmetry(self, mesh: trimesh.Trimesh) -> bool: | |
| """Check if mesh is roughly symmetric""" | |
| # Simple check: compare left and right halves | |
| vertices = mesh.vertices | |
| center_x = mesh.centroid[0] | |
| left_verts = vertices[vertices[:, 0] < center_x] | |
| right_verts = vertices[vertices[:, 0] > center_x] | |
| # Check if similar number of vertices on each side | |
| ratio = len(left_verts) / (len(right_verts) + 1) | |
| return 0.8 < ratio < 1.2 | |
| def _detect_limbs(self, mesh: trimesh.Trimesh) -> List[Dict]: | |
| """Detect potential limbs in the mesh""" | |
| # Simplified limb detection using vertex clustering | |
| from sklearn.cluster import DBSCAN | |
| limbs = [] | |
| try: | |
| # Cluster vertices to find distinct parts | |
| clustering = DBSCAN(eps=0.1, min_samples=10).fit(mesh.vertices) | |
| # Analyze each cluster | |
| for label in set(clustering.labels_): | |
| if label == -1: # Noise | |
| continue | |
| cluster_verts = mesh.vertices[clustering.labels_ == label] | |
| # Check if cluster could be a limb | |
| cluster_bounds = np.array([cluster_verts.min(axis=0), cluster_verts.max(axis=0)]) | |
| dimensions = cluster_bounds[1] - cluster_bounds[0] | |
| # Limbs are typically elongated | |
| if max(dimensions) / (min(dimensions) + 0.001) > 2: | |
| limbs.append({ | |
| 'center': cluster_verts.mean(axis=0), | |
| 'direction': dimensions, | |
| 'size': len(cluster_verts) | |
| }) | |
| except: | |
| # Fallback if clustering fails | |
| pass | |
| return limbs | |
| def _classify_mesh_type(self, mesh: trimesh.Trimesh) -> str: | |
| """Classify the type of creature mesh""" | |
| analysis = { | |
| 'height': mesh.bounds[1][2] - mesh.bounds[0][2], | |
| 'width': mesh.bounds[1][0] - mesh.bounds[0][0], | |
| 'depth': mesh.bounds[1][1] - mesh.bounds[0][1] | |
| } | |
| # Simple classification based on proportions | |
| aspect_ratio = analysis['height'] / max(analysis['width'], analysis['depth']) | |
| if aspect_ratio > 1.5: | |
| return 'bipedal' # Tall creatures | |
| elif aspect_ratio < 0.7: | |
| return 'quadruped' # Wide creatures | |
| else: | |
| return 'hybrid' # Mixed proportions | |
| def _generate_skeleton(self, mesh: trimesh.Trimesh, analysis: Dict) -> Dict: | |
| """Generate skeleton for the mesh""" | |
| skeleton = { | |
| 'bones': [], | |
| 'hierarchy': {}, | |
| 'bind_poses': [] | |
| } | |
| # Create root bone at center | |
| root_pos = analysis['center'] | |
| root_bone = { | |
| 'id': 0, | |
| 'name': 'root', | |
| 'position': root_pos, | |
| 'parent': -1, | |
| 'children': [] | |
| } | |
| skeleton['bones'].append(root_bone) | |
| # Generate bones based on mesh type | |
| mesh_type = analysis['mesh_type'] | |
| if mesh_type == 'bipedal': | |
| skeleton = self._generate_bipedal_skeleton(mesh, skeleton, analysis) | |
| elif mesh_type == 'quadruped': | |
| skeleton = self._generate_quadruped_skeleton(mesh, skeleton, analysis) | |
| else: | |
| skeleton = self._generate_hybrid_skeleton(mesh, skeleton, analysis) | |
| # Build hierarchy | |
| for bone in skeleton['bones']: | |
| if bone['parent'] >= 0: | |
| skeleton['bones'][bone['parent']]['children'].append(bone['id']) | |
| return skeleton | |
| def _generate_bipedal_skeleton(self, mesh: trimesh.Trimesh, skeleton: Dict, analysis: Dict) -> Dict: | |
| """Generate skeleton for bipedal creature""" | |
| bounds = analysis['bounds'] | |
| center = analysis['center'] | |
| height = analysis['height'] | |
| # Spine bones | |
| spine_positions = [ | |
| center + [0, 0, -height * 0.4], # Hips | |
| center + [0, 0, 0], # Chest | |
| center + [0, 0, height * 0.3] # Head | |
| ] | |
| parent_id = 0 | |
| for i, pos in enumerate(spine_positions): | |
| bone = { | |
| 'id': len(skeleton['bones']), | |
| 'name': ['hips', 'chest', 'head'][i], | |
| 'position': pos, | |
| 'parent': parent_id, | |
| 'children': [] | |
| } | |
| skeleton['bones'].append(bone) | |
| parent_id = bone['id'] | |
| # Add limbs | |
| chest_id = skeleton['bones'][2]['id'] # Chest bone | |
| hips_id = skeleton['bones'][1]['id'] # Hips bone | |
| # Arms | |
| arm_offset = analysis['width'] * 0.4 | |
| for side, offset in [('left', -arm_offset), ('right', arm_offset)]: | |
| shoulder_pos = skeleton['bones'][chest_id]['position'] + [offset, 0, 0] | |
| elbow_pos = shoulder_pos + [offset * 0.5, 0, -height * 0.2] | |
| # Shoulder | |
| shoulder = { | |
| 'id': len(skeleton['bones']), | |
| 'name': f'{side}_shoulder', | |
| 'position': shoulder_pos, | |
| 'parent': chest_id, | |
| 'children': [] | |
| } | |
| skeleton['bones'].append(shoulder) | |
| # Elbow/Hand | |
| hand = { | |
| 'id': len(skeleton['bones']), | |
| 'name': f'{side}_hand', | |
| 'position': elbow_pos, | |
| 'parent': shoulder['id'], | |
| 'children': [] | |
| } | |
| skeleton['bones'].append(hand) | |
| # Legs | |
| for side, offset in [('left', -arm_offset * 0.5), ('right', arm_offset * 0.5)]: | |
| hip_pos = skeleton['bones'][hips_id]['position'] + [offset, 0, 0] | |
| foot_pos = hip_pos + [0, 0, -height * 0.4] | |
| # Leg | |
| leg = { | |
| 'id': len(skeleton['bones']), | |
| 'name': f'{side}_leg', | |
| 'position': hip_pos, | |
| 'parent': hips_id, | |
| 'children': [] | |
| } | |
| skeleton['bones'].append(leg) | |
| # Foot | |
| foot = { | |
| 'id': len(skeleton['bones']), | |
| 'name': f'{side}_foot', | |
| 'position': foot_pos, | |
| 'parent': leg['id'], | |
| 'children': [] | |
| } | |
| skeleton['bones'].append(foot) | |
| return skeleton | |
| def _generate_quadruped_skeleton(self, mesh: trimesh.Trimesh, skeleton: Dict, analysis: Dict) -> Dict: | |
| """Generate skeleton for quadruped creature""" | |
| # Similar to bipedal but with 4 legs and horizontal spine | |
| center = analysis['center'] | |
| width = analysis['width'] | |
| depth = analysis['depth'] | |
| # Spine (horizontal) | |
| spine_positions = [ | |
| center + [-width * 0.3, 0, 0], # Tail | |
| center, # Body | |
| center + [width * 0.3, 0, 0] # Head | |
| ] | |
| parent_id = 0 | |
| for i, pos in enumerate(spine_positions): | |
| bone = { | |
| 'id': len(skeleton['bones']), | |
| 'name': ['tail', 'body', 'head'][i], | |
| 'position': pos, | |
| 'parent': parent_id, | |
| 'children': [] | |
| } | |
| skeleton['bones'].append(bone) | |
| parent_id = bone['id'] if i < 2 else skeleton['bones'][1]['id'] | |
| # Add 4 legs | |
| body_id = skeleton['bones'][1]['id'] | |
| for front_back, x_offset in [('front', width * 0.2), ('back', -width * 0.2)]: | |
| for side, z_offset in [('left', -depth * 0.3), ('right', depth * 0.3)]: | |
| leg_pos = skeleton['bones'][body_id]['position'] + [x_offset, -analysis['height'] * 0.3, z_offset] | |
| leg = { | |
| 'id': len(skeleton['bones']), | |
| 'name': f'{front_back}_{side}_leg', | |
| 'position': leg_pos, | |
| 'parent': body_id, | |
| 'children': [] | |
| } | |
| skeleton['bones'].append(leg) | |
| return skeleton | |
| def _generate_hybrid_skeleton(self, mesh: trimesh.Trimesh, skeleton: Dict, analysis: Dict) -> Dict: | |
| """Generate skeleton for hybrid creature""" | |
| # Mix of bipedal and quadruped features | |
| # For simplicity, use bipedal as base | |
| return self._generate_bipedal_skeleton(mesh, skeleton, analysis) | |
| def _compute_bone_weights(self, mesh: trimesh.Trimesh, skeleton: Dict) -> np.ndarray: | |
| """Compute bone weights for vertices""" | |
| num_vertices = len(mesh.vertices) | |
| num_bones = len(skeleton['bones']) | |
| # Initialize weights matrix | |
| weights = np.zeros((num_vertices, num_bones)) | |
| # For each vertex, compute influence from each bone | |
| for v_idx, vertex in enumerate(mesh.vertices): | |
| total_weight = 0 | |
| for b_idx, bone in enumerate(skeleton['bones']): | |
| # Distance-based weight | |
| distance = np.linalg.norm(vertex - bone['position']) | |
| # Inverse distance weight with falloff | |
| weight = 1.0 / (distance + 0.1) | |
| weights[v_idx, b_idx] = weight | |
| total_weight += weight | |
| # Normalize weights | |
| if total_weight > 0: | |
| weights[v_idx] /= total_weight | |
| # Keep only top 4 influences per vertex (standard for game engines) | |
| top_4 = np.argsort(weights[v_idx])[-4:] | |
| mask = np.zeros(num_bones, dtype=bool) | |
| mask[top_4] = True | |
| weights[v_idx, ~mask] = 0 | |
| # Re-normalize | |
| if weights[v_idx].sum() > 0: | |
| weights[v_idx] /= weights[v_idx].sum() | |
| return weights | |
| def _create_default_animations(self, skeleton: Dict) -> Dict[str, List]: | |
| """Create default animations for the skeleton""" | |
| animations = {} | |
| # Create basic animation sets | |
| for anim_name, anim_func in self.animation_presets.items(): | |
| animations[anim_name] = anim_func(skeleton) | |
| return animations | |
| def _create_idle_animation(self, skeleton: Dict) -> List[Dict]: | |
| """Create idle animation keyframes""" | |
| keyframes = [] | |
| # Simple breathing/bobbing motion | |
| for t in np.linspace(0, 2 * np.pi, 30): | |
| frame = { | |
| 'time': t / (2 * np.pi), | |
| 'bones': {} | |
| } | |
| # Subtle movement for each bone | |
| for bone in skeleton['bones']: | |
| if 'chest' in bone['name'] or 'body' in bone['name']: | |
| # Breathing motion | |
| offset = np.sin(t) * 0.02 | |
| frame['bones'][bone['id']] = { | |
| 'position': bone['position'] + [0, offset, 0], | |
| 'rotation': [0, 0, 0, 1] # Quaternion | |
| } | |
| else: | |
| # No movement | |
| frame['bones'][bone['id']] = { | |
| 'position': bone['position'], | |
| 'rotation': [0, 0, 0, 1] | |
| } | |
| keyframes.append(frame) | |
| return keyframes | |
| def _create_walk_animation(self, skeleton: Dict) -> List[Dict]: | |
| """Create walk animation keyframes""" | |
| # Simplified walk cycle | |
| keyframes = [] | |
| for t in np.linspace(0, 2 * np.pi, 60): | |
| frame = { | |
| 'time': t / (2 * np.pi), | |
| 'bones': {} | |
| } | |
| # Animate legs with sine waves | |
| for bone in skeleton['bones']: | |
| if 'leg' in bone['name'] or 'foot' in bone['name']: | |
| # Alternating leg movement | |
| phase = 0 if 'left' in bone['name'] else np.pi | |
| offset = np.sin(t + phase) * 0.1 | |
| frame['bones'][bone['id']] = { | |
| 'position': bone['position'] + [offset, 0, 0], | |
| 'rotation': [0, 0, 0, 1] | |
| } | |
| else: | |
| frame['bones'][bone['id']] = { | |
| 'position': bone['position'], | |
| 'rotation': [0, 0, 0, 1] | |
| } | |
| keyframes.append(frame) | |
| return keyframes | |
| def _create_attack_animation(self, skeleton: Dict) -> List[Dict]: | |
| """Create attack animation keyframes""" | |
| # Quick strike motion | |
| keyframes = [] | |
| # Wind up | |
| for t in np.linspace(0, 0.3, 10): | |
| frame = {'time': t, 'bones': {}} | |
| for bone in skeleton['bones']: | |
| frame['bones'][bone['id']] = { | |
| 'position': bone['position'], | |
| 'rotation': [0, 0, 0, 1] | |
| } | |
| keyframes.append(frame) | |
| # Strike | |
| for t in np.linspace(0.3, 0.5, 5): | |
| frame = {'time': t, 'bones': {}} | |
| for bone in skeleton['bones']: | |
| if 'hand' in bone['name'] or 'head' in bone['name']: | |
| # Forward motion | |
| offset = (t - 0.3) * 0.5 | |
| frame['bones'][bone['id']] = { | |
| 'position': bone['position'] + [offset, 0, 0], | |
| 'rotation': [0, 0, 0, 1] | |
| } | |
| else: | |
| frame['bones'][bone['id']] = { | |
| 'position': bone['position'], | |
| 'rotation': [0, 0, 0, 1] | |
| } | |
| keyframes.append(frame) | |
| # Return | |
| for t in np.linspace(0.5, 1.0, 10): | |
| frame = {'time': t, 'bones': {}} | |
| for bone in skeleton['bones']: | |
| frame['bones'][bone['id']] = { | |
| 'position': bone['position'], | |
| 'rotation': [0, 0, 0, 1] | |
| } | |
| keyframes.append(frame) | |
| return keyframes | |
| def _create_happy_animation(self, skeleton: Dict) -> List[Dict]: | |
| """Create happy/excited animation keyframes""" | |
| # Jumping or bouncing motion | |
| keyframes = [] | |
| for t in np.linspace(0, 2 * np.pi, 40): | |
| frame = { | |
| 'time': t / (2 * np.pi), | |
| 'bones': {} | |
| } | |
| # Bouncing motion | |
| bounce = abs(np.sin(t * 2)) * 0.1 | |
| for bone in skeleton['bones']: | |
| frame['bones'][bone['id']] = { | |
| 'position': bone['position'] + [0, bounce, 0], | |
| 'rotation': [0, 0, 0, 1] | |
| } | |
| keyframes.append(frame) | |
| return keyframes | |
| def _save_rigged_model(self, rigged_model: Dict) -> str: | |
| """Save rigged model to file""" | |
| # Create temporary file | |
| with tempfile.NamedTemporaryFile(suffix='.glb', delete=False) as tmp: | |
| output_path = tmp.name | |
| # In production, this would export the rigged model with animations | |
| # For now, just save the mesh | |
| rigged_model['mesh'].export(output_path) | |
| return output_path | |
| def _save_mesh_without_rigging(self, mesh: Union[str, trimesh.Trimesh]) -> str: | |
| """Save mesh without rigging as fallback""" | |
| if isinstance(mesh, str): | |
| return mesh | |
| with tempfile.NamedTemporaryFile(suffix='.glb', delete=False) as tmp: | |
| output_path = tmp.name | |
| mesh.export(output_path) | |
| return output_path | |
| def to(self, device: str): | |
| """Move model to specified device (compatibility method)""" | |
| self.device = device |