feat: Support skeletal animation basics [flame_3d]

packages/flame_3d/lib/src/graphics/graphics_device.dart

@@ -3,6 +3,7 @@ import 'dart:ui';
 
 import 'package:flame_3d/game.dart';
 import 'package:flame_3d/resources.dart';
+import 'package:flame_3d/src/graphics/joints_info.dart';
 import 'package:flutter_gpu/gpu.dart' as gpu;
 
 enum BlendState {
@@ -48,6 +49,10 @@ class GraphicsDevice {
 
   Size _previousSize = Size.zero;
 
+  /// Must be set by the rendering pipeline before elements are bound.
+  /// Can be accessed by elements in their bind method.
+  final JointsInfo jointsInfo = JointsInfo();
+
   /// Must be set by the rendering pipeline before elements are bound.
   /// Can be accessed by elements in their bind method.
   final LightingInfo lightingInfo = LightingInfo();

packages/flame_3d/lib/src/graphics/joints_info.dart

@@ -0,0 +1,13 @@
+import 'package:flame_3d/core.dart';
+
+class JointsInfo {
+  /// Joints per surface idx
+  Map<int, List<Matrix4>> jointTransformsPerSurface = {};
+
+  /// Joints for the current surface
+  List<Matrix4> jointTransforms = [];
+
+  void setSurface(int surfaceIdx) {
+    jointTransforms = jointTransformsPerSurface[surfaceIdx] ?? [];
+  }
+}

packages/flame_3d/lib/src/resources/material/spatial_material.dart

@@ -16,7 +16,18 @@ class SpatialMaterial extends Material {
           vertexShader: Shader(
             _library['TextureVertex']!,
             slots: [
-              UniformSlot.value('VertexInfo', {'model', 'view', 'projection'}),
+              UniformSlot.value('VertexInfo', {
+                'model',
+                'view',
+                'projection',
+              }),
+              UniformSlot.value('JointMatrices', {
+                'joint0',
+                'joint1',
+                'joint2',
+                'joint3',
+                'joint4',
+              }),
             ],
           ),
           fragmentShader: Shader(
@@ -56,6 +67,7 @@ class SpatialMaterial extends Material {
   @override
   void bind(GraphicsDevice device) {
     _bindVertexInfo(device);
+    _bindJointMatrices(device);
     _bindMaterial(device);
     _bindCamera(device);
   }
@@ -67,6 +79,18 @@ class SpatialMaterial extends Material {
       ..setMatrix4('VertexInfo.projection', device.projection);
   }
 
+  void _bindJointMatrices(GraphicsDevice device) {
+    final jointTransforms = device.jointsInfo.jointTransforms;
+    if (jointTransforms.length > 5) {
+      throw Exception(
+        'At most 5 joints per surface, found ${jointTransforms.length}',
+      );
+    }
+    for (final (idx, transform) in jointTransforms.indexed) {
+      vertexShader.setMatrix4('JointMatrices.joint$idx', transform);
+    }
+  }
+
   void _bindMaterial(GraphicsDevice device) {
     _applyLights(device);
     fragmentShader

packages/flame_3d/lib/src/resources/mesh/mesh.dart

@@ -26,7 +26,8 @@ class Mesh extends Resource<void> {
   int get vertexCount => _surfaces.fold(0, (p, e) => p + e.vertexCount);
 
   void bind(GraphicsDevice device) {
-    for (final surface in _surfaces) {
+    for (final (idx, surface) in _surfaces.indexed) {
+      device.jointsInfo.setSurface(idx);
       device.bindSurface(surface);
     }
   }

packages/flame_3d/lib/src/resources/mesh/surface.dart

@@ -18,6 +18,7 @@ class Surface extends Resource<gpu.DeviceBuffer?> {
     required List<Vertex> vertices,
     required List<int> indices,
     this.material,
+    this.jointMap,
     /**
      * If `true`, the normals will be calculated if they are not provided.
      */
@@ -34,15 +35,16 @@ class Surface extends Resource<gpu.DeviceBuffer?> {
     _vertices = Float32List.fromList(
       normalizedVertices.fold([], (p, v) => p..addAll(v.storage)),
     ).buffer;
-    _vertexCount = _vertices.lengthInBytes ~/ (normalizedVertices.length * 9);
+    _vertexCount = normalizedVertices.length;
 
     _indices = Uint16List.fromList(indices).buffer;
-    _indexCount = _indices.lengthInBytes ~/ 2;
+    _indexCount = indices.length;
 
     _calculateAabb(normalizedVertices);
   }
 
   Material? material;
+  Map<int, int>? jointMap;
 
   Aabb3 get aabb => _aabb;
   late Aabb3 _aabb;

packages/flame_3d/lib/src/resources/mesh/vertex.dart

@@ -18,14 +18,20 @@ class Vertex {
     required Vector2 texCoord,
     this.color = const Color(0xFFFFFFFF),
     Vector3? normal,
+    Vector4? joints,
+    Vector4? weights,
   })  : position = position.immutable,
         texCoord = texCoord.immutable,
         normal = normal?.immutable,
+        joints = joints?.immutable,
+        weights = weights?.immutable,
         _storage = Float32List.fromList([
           ...position.storage, // 1, 2, 3
           ...texCoord.storage, // 4, 5
           ...color.storage, // 6, 7, 8, 9
           ...(normal ?? Vector3.zero()).storage, // 10, 11, 12
+          ...(joints ?? Vector4.zero()).storage, // 13, 14, 15, 16
+          ...(weights ?? Vector4.zero()).storage, // 17, 18, 19, 20
         ]);
 
   Float32List get storage => _storage;
@@ -40,6 +46,12 @@ class Vertex {
   /// The normal vector of the vertex.
   final ImmutableVector3? normal;
 
+  /// The joints of the vertex.
+  final ImmutableVector4? joints;
+
+  /// The weights of the vertex.
+  final ImmutableVector4? weights;
+
   /// The color on the vertex.
   final Color color;
 
@@ -49,23 +61,36 @@ class Vertex {
       position == other.position &&
       texCoord == other.texCoord &&
       normal == other.normal &&
-      color == other.color;
+      color == other.color &&
+      joints == other.joints &&
+      weights == other.weights;
 
   @override
-  int get hashCode => Object.hashAll([position, texCoord, normal, color]);
+  int get hashCode => Object.hashAll([
+        position,
+        texCoord,
+        normal,
+        color,
+        joints,
+        weights,
+      ]);
 
   Vertex copyWith({
     Vector3? position,
     Vector2? texCoord,
     Vector3? normal,
     Color? color,
+    Vector4? joints,
+    Vector4? weights,
   }) {
     // TODO(wolfenrain): optimize this.
     return Vertex(
       position: position ?? this.position.mutable,
       texCoord: texCoord ?? this.texCoord.mutable,
       normal: normal ?? this.normal?.mutable,
       color: color ?? this.color,
+      joints: joints ?? this.joints?.mutable,
+      weights: weights ?? this.weights?.mutable,
     );
   }
 

packages/flame_3d/shaders/spatial_material.vert

@@ -4,6 +4,8 @@ in vec3 vertexPosition;
 in vec2 vertexTexCoord;
 in vec4 vertexColor;
 in vec3 vertexNormal;
+in vec4 vertexJoints;
+in vec4 vertexWeights;
 
 out vec2 fragTexCoord;
 out vec4 fragColor;
@@ -16,18 +18,58 @@ uniform VertexInfo {
   mat4 projection;
 } vertex_info;
 
+uniform JointMatrices {
+  mat4 joint0;
+  mat4 joint1;
+  mat4 joint2;
+  mat4 joint3;
+  mat4 joint4;
+} joints;
+
+mat4 jointMat(float jointIndex) {
+  if (jointIndex == 0.0) {
+    return joints.joint0;
+  } else if (jointIndex == 1.0) {
+    return joints.joint1;
+  } else if (jointIndex == 2.0) {
+    return joints.joint2;
+  } else if (jointIndex == 3.0) {
+    return joints.joint3;
+  } else if (jointIndex == 4.0) {
+    return joints.joint4;
+  } else {
+    return mat4(0.0);
+  }
+}
+
+mat4 computeSkinMatrix() {
+  if (vertexWeights.x == 0.0 && vertexWeights.y == 0.0 && vertexWeights.z == 0.0 && vertexWeights.w == 0.0) {
+    // no weights, skip skinning
+    return mat4(1.0);
+  }
+
+  return vertexWeights.x * jointMat(vertexJoints.x) +
+    vertexWeights.y * jointMat(vertexJoints.y) +
+    vertexWeights.z * jointMat(vertexJoints.z) +
+    vertexWeights.w * jointMat(vertexJoints.w);
+}
+
 void main() {
+  mat4 skinMatrix = computeSkinMatrix();
+  vec3 position = (skinMatrix * vec4(vertexPosition, 1.0)).xyz;
+  vec3 normal = normalize((skinMatrix * vec4(vertexNormal, 0.0)).xyz);
+
   // Calculate the modelview projection matrix
   mat4 modelViewProjection = vertex_info.projection * vertex_info.view * vertex_info.model;
 
   // Transform the vertex position
-  gl_Position = modelViewProjection * vec4(vertexPosition, 1.0);
+  gl_Position = modelViewProjection * vec4(position, 1.0);
 
   // Pass the interpolated values to the fragment shader
   fragTexCoord = vertexTexCoord;
   fragColor = vertexColor;
 
   // Calculate the world-space position and normal
-  fragPosition = vec3(vertex_info.model * vec4(vertexPosition, 1.0));
-  fragNormal = mat3(transpose(inverse(vertex_info.model))) * vertexNormal;
+  fragPosition = vec3(vertex_info.model * vec4(position, 1.0));
+  fragNormal = mat3(transpose(inverse(vertex_info.model))) * normal;
 }