Creating a voxel mesh and render it using metal within a RealityKit ImmersiveView

Look at using these in your modelComponent: MeshResource, MeshResource.Contents, MeshResource.Part

e.g.

func meshContents() ->  MeshResource.Contents {
       // update mesh data
        …
        var meshPart = MeshResource.Part(…)
        meshPart.positions = .init(positions)
        meshPart.textureCoordinates = MeshBuffer(textureCoordinates)
        meshPart.triangleIndices = .init(indices)

        var contents = MeshResource.Contents()
        contents.models = [.init(…, parts: [meshPart])]

         return contents
}

let meshResource = try await MeshResource(from: meshContents())
let modelComponent = ModelComponent(mesh: meshResource, …)

for loop {
            try modelComponent.mesh.replace(with: meshContents())
}

@deeje thanks for your reply. I did a bit of research and it seems this approach cannot be really scaled for millions of voxels, right? I need to update the opacity of each individual voxel, and it seems your suggested approach requires a regeneration of the mesh each time. I tried nevertheless, but I didn't manage to go far since, again, I can't really find proper documentation on how to do this. This is what I got so far:

//  VoxelMesh.swift
//  arcade
//
//  Created by Alejandro Marcos Aragón on 06/06/2025.
//

import Foundation
import RealityKit
import SwiftUI


// Define Voxel struct
struct Voxel {
    var position: SIMD3<Float>
    var color: SIMD4<Float>
}

// Function to create cube geometry
func makeCube(at position: SIMD3<Float>, size: Float) -> (vertices: [SIMD3<Float>], normals: [SIMD3<Float>], indices: [UInt32]) {
    let halfSize = size / 2
    let vertices: [SIMD3<Float>] = [
        // Front face
        [-halfSize, -halfSize,  halfSize], // 0
        [ halfSize, -halfSize,  halfSize], // 1
        [ halfSize,  halfSize,  halfSize], // 2
        [-halfSize,  halfSize,  halfSize], // 3
        // Back face
        [-halfSize, -halfSize, -halfSize], // 4
        [ halfSize, -halfSize, -halfSize], // 5
        [ halfSize,  halfSize, -halfSize], // 6
        [-halfSize,  halfSize, -halfSize]  // 7
    ].map { $0 + position } // Offset by voxel position
    
    let normals: [SIMD3<Float>] = [
        [ 0,  0,  1], // Front
        [ 0,  0,  1],
        [ 0,  0,  1],
        [ 0,  0,  1],
        [ 0,  0, -1], // Back
        [ 0,  0, -1],
        [ 0,  0, -1],
        [ 0,  0, -1]
    ]
    
    let indices: [UInt32] = [
        // Front
        0, 1, 2,  2, 3, 0,
        // Right
        1, 5, 6,  6, 2, 1,
        // Back
        5, 4, 7,  7, 6, 5,
        // Left
        4, 0, 3,  3, 7, 4,
        // Top
        3, 2, 6,  6, 7, 3,
        // Bottom
        0, 4, 5,  5, 1, 0
    ]
    
    return (vertices, normals, indices)
}

// Function to create voxel mesh contents
func makeVoxelMeshContents(voxels: [Voxel], voxelSize: Float = 1.0) throws -> MeshResource.Contents {
    var positions: [SIMD3<Float>] = []
    var normals: [SIMD3<Float>] = []
    var colors: [SIMD4<Float>] = []
    var indices: [UInt32] = []
    var uvs: [SIMD2<Float>] = []
    
    // Reserve capacity for performance
    let vertexCountPerCube = 8
    let indexCountPerCube = 36
    positions.reserveCapacity(voxels.count * vertexCountPerCube)
    normals.reserveCapacity(voxels.count * vertexCountPerCube)
    colors.reserveCapacity(voxels.count * vertexCountPerCube)
    indices.reserveCapacity(voxels.count * indexCountPerCube)
    
    var vertexOffset: UInt32 = 0
    
    // Generate geometry for each voxel
    for voxel in voxels {
        let (verts, norms, inds) = makeCube(at: voxel.position, size: voxelSize)
        positions.append(contentsOf: verts)
        normals.append(contentsOf: norms)
        // Repeat or generate UVs per vertex of the cube
        let cubeUVs: [SIMD2<Float>] = [
            [0, 0], [1, 0], [1, 1], [0, 1], // front face
            [0, 0], [1, 0], [1, 1], [0, 1]  // back face
        ]
        uvs.append(contentsOf: cubeUVs)
        colors.append(contentsOf: Array(repeating: voxel.color, count: verts.count))
        indices.append(contentsOf: inds.map { $0 + vertexOffset })
        vertexOffset += UInt32(verts.count)
    }
    
    // Create MeshDescriptor
    var descriptor = MeshDescriptor(name: "voxels")

    descriptor.positions = MeshBuffer(positions)
    descriptor.normals = MeshBuffer(normals)
    descriptor.textureCoordinates = MeshBuffer(uvs)
//    descriptor.colors = MeshBuffer(colors)

    descriptor.primitives = .triangles(indices)
    descriptor.materials = .allFaces(0)

    // Generate MeshResource and return its contents
    let mesh = try MeshResource.generate(from: [descriptor])
    return mesh.contents
}


class VoxelMesh: ObservableObject {
    @Published var voxels: [Voxel] = []
    var entity: ModelEntity?
    var elSize: Float = 1.0
    var parentEntity: Entity
    private var isInitialized = false
    
    init(parent: Entity, voxels: [Voxel], voxelSize: Float = 1.0) {
        self.parentEntity = parent
        self.voxels = voxels
        self.elSize = voxelSize
        Task {
            await self.generateVoxelEntity()
            await MainActor.run {
                self.isInitialized = true // Mark as initialized
            }
        }
    }
    func waitForInitialization() async {
        while !isInitialized {
            try? await Task.sleep(nanoseconds: 100_000_000) // Wait 0.1 seconds
        }
    }

    func generateVoxelEntity() async {
        do {
            let contents = try makeVoxelMeshContents(voxels: voxels, voxelSize: elSize)
            let mesh = try await MeshResource(from: contents)

            guard let device = MTLCreateSystemDefaultDevice(),
                  let library = try? device.makeDefaultLibrary(bundle: .main) else {
                fatalError("Failed to load default Metal library.")
            }

            let entity = await MainActor.run {
                return ModelEntity(
                    mesh: mesh,
                    materials: [SimpleMaterial(color: .white, isMetallic: false)]
                )
            }
            
            await entity.generateCollisionShapes(recursive: false)
            self.entity = entity
            parentEntity.addChild(entity)
            await MainActor.run {
                self.isInitialized = true
            }
        } catch {
            print("Failed to generate voxel mesh: \(error)")
        }
    }
    
    func updateVoxelOpacities(newOpacities: [Double]) async {
        
        print("3. lalalalal")
        guard newOpacities.count == voxels.count else {
            fatalError("Number of opacity values is not the same as the number of mesh voxels")
        }

        await MainActor.run {
            for i in voxels.indices {
                var voxel = voxels[i]
                voxel.color.w = Float(newOpacities[i])
                voxels[i] = voxel
            }
        }
                
        do {
            let updatedContents = try makeVoxelMeshContents(voxels: voxels, voxelSize: elSize)
            let newMesh = try await MeshResource(from: updatedContents)
            self.entity?.model?.mesh = newMesh
        } catch {
            print("Failed to update voxel mesh: \(error)")
        }
    }
}

It seems there are many issues behind this code, including the fact hat I can't use a SimpleMaterial to assign different parity values to all voxels.

I changed the material to

materials: [UnlitMaterial(color: UIColor.clear.withAlphaComponent(0.5))]

to no avail, I simply see a black box that doesn't change opacity.