public func convert2VUYToRGB(
cvPixelBuffer: CVPixelBuffer,
device: MTLDevice
) throws -> MTLTexture {
// Create a Metal texture cache if not already created
var textureCache: CVMetalTextureCache?
guard CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, device, nil, &textureCache) == kCVReturnSuccess,
let cache = textureCache else {
throw MetalScalingErrors.failedToCreateTextureCache
}
// Lock the base address of the CVPixelBuffer
guard kCVReturnSuccess == CVPixelBufferLockBaseAddress(cvPixelBuffer, .readOnly) else {
throw MetalScalingErrors.failedToLockPixelBuffer
}
defer {
// Always unlock the CVPixelBuffer
CVPixelBufferUnlockBaseAddress(cvPixelBuffer, .readOnly)
}
// Create a Metal texture for the interleaved YUV data
let yuvTexture = try createMTLTextureForPlane(
cvPixelBuffer: cvPixelBuffer,
planeIndex: 0, // Only one plane for 2vuy
textureCache: cache,
format: .gbgr422, // Adjust format as needed for your shader
device: device)
// Create a Metal texture for RGB output
let rgbTextureDescriptor = MTLTextureDescriptor.texture2DDescriptor(
pixelFormat: .rgba8Unorm,
width: CVPixelBufferGetWidth(cvPixelBuffer),
height: CVPixelBufferGetHeight(cvPixelBuffer),
mipmapped: false)
rgbTextureDescriptor.usage = [.shaderRead, .shaderWrite]
guard let rgbTexture = device.makeTexture(descriptor: rgbTextureDescriptor) else {
throw MetalScalingErrors.failedToCreateTexture
}
// Create a Metal compute pipeline with a shader that converts YUV to RGB
if twoVUYToRgbKernelPipeline == nil {
twoVUYToRgbKernelPipeline = try createComputePipeline(device: device, shaderName: "twoVUYToRgb")
}
// Set up a command buffer and encoder
guard let commandQueue = device.makeCommandQueue(),
let commandBuffer = commandQueue.makeCommandBuffer(),
let computeEncoder = commandBuffer.makeComputeCommandEncoder() else {
throw MetalScalingErrors.errorSettingUpEncoder
}
guard let twoVUYToRgbKernelPipeline = twoVUYToRgbKernelPipeline else {
throw MetalScalingErrors.failedToCreatePipeline
}
// Set textures and encode the compute shader
computeEncoder.setComputePipelineState(twoVUYToRgbKernelPipeline)
computeEncoder.setTexture(yuvTexture, index: 0) // Use the interleaved YUV texture
computeEncoder.setTexture(rgbTexture, index: 1) // Output RGB texture
// Calculate threadgroup and grid sizes
let threadgroupCount = MTLSize(width: 8, height: 8, depth: 1)
let threadsPerThreadgroup = MTLSize(
width: (rgbTexture.width + threadgroupCount.width - 1) / threadgroupCount.width,
height: (rgbTexture.height + threadgroupCount.height - 1) / threadgroupCount.height,
depth: 1)
defer {
computeEncoder.endEncoding()
commandBuffer.commit()
}
computeEncoder.dispatchThreadgroups(threadsPerThreadgroup, threadsPerThreadgroup: threadgroupCount)
// Return the RGB texture
return rgbTexture
}
public func createMTLTextureForPlane(
cvPixelBuffer: CVPixelBuffer,
planeIndex: Int,
textureCache: CVMetalTextureCache,
format: MTLPixelFormat,
device: MTLDevice
) throws -> MTLTexture {
// Create a Metal texture from the CVPixelBuffer plane
let width = CVPixelBufferGetWidthOfPlane(cvPixelBuffer, planeIndex)
let height = CVPixelBufferGetHeightOfPlane(cvPixelBuffer, planeIndex)
var cvTexture: CVMetalTexture?
guard CVMetalTextureCacheCreateTextureFromImage(
kCFAllocatorDefault,
textureCache,
cvPixelBuffer,
nil,
format,
width,
height,
planeIndex,
&cvTexture) == kCVReturnSuccess else {
throw MetalScalingErrors.failedToCreateTextureCacheFromImage
}
if let cache = cvTexture, let metalTexture = CVMetalTextureGetTexture(cache) {
return metalTexture
} else {
throw MetalScalingErrors.failedToGetTexture
}
}
// Metal Kernal
kernel void twoVUYToRgb(texture2d yuvTexture [[texture(0)]],
texture2d rgbTexture [[texture(1)]],
uint2 gid [[thread_position_in_grid]]) {
// Get the width and height of the texture
uint width = yuvTexture.get_width();
uint height = yuvTexture.get_height();
// Ensure the thread is within the bounds of the texture
if (gid.x >= width || gid.y >= height) {
return; // Exit if out of bounds
}
// Read the Y value (luminance) from the red channel of the texture
float y = yuvTexture.read(gid).r;
// Calculate the corresponding index for the U and V samples (subsampled by 2)
uint uvX = gid.x / 2; // Divide by 2 for horizontal subsampling
uint uvY = gid.y / 2; // Divide by 2 for vertical subsampling
// Read the interleaved U and V values (U in green, V in blue)
float2 uv = yuvTexture.read(uint2(uvX, uvY)).gb;
float u = uv.x - 0.5; // Offset U by 0.5 for correct color range
float v = uv.y - 0.5; // Offset V by 0.5 for correct color range
// BT.709 YUV to RGB conversion (for HD content)
float r = y + 1.5748 * v;
float g = y - 0.1873 * u - 0.4681 * v;
float b = y + 1.8556 * u;
// Clamp the RGB values to the range [0, 1]
r = clamp(r, 0.0, 1.0);
g = clamp(g, 0.0, 1.0);
b = clamp(b, 0.0, 1.0);
// Write the RGB values to the output texture
rgbTexture.write(float4(r, g, b, 1.0), gid);
}