Idea: Metal Hello world in Kotlin

[soywiz]

import kotlinx.cinterop.*
import platform.CoreGraphics.*
import platform.Foundation.*
import platform.Metal.*
import platform.QuartzCore.*
import platform.posix.*

// https://www.kodeco.com/7475-metal-tutorial-getting-started
fun main(): Unit = memScoped {
    val width = 50
    val height = 50

    CGImageCreateCopy(null) // Need to link Core Graphics
    //CGImageCreate(100, 100)
    val device = MTLCreateSystemDefaultDevice() ?: error("Can't find Metal Device")
    println("device: $device")
    val layer = CAMetalLayer()
    layer.device = device
    layer.pixelFormat = MTLPixelFormatBGRA8Unorm
    //layer.framebufferOnly = true
    layer.framebufferOnly = false
    layer.frame = CGRectMake(0.0, 0.0, width.toDouble(), height.toDouble())
    val data = floatArrayOf(0.0f,  1.0f, 0.0f, -1.0f, -1.0f, 0.0f, 1.0f, -1.0f, 0.0f)
    val vertexBuffer: MTLBufferProtocol = device.newBufferWithLength((data.size * Float.SIZE_BYTES).convert(), 0) ?: error("Can't create buffer!")
    data.usePinned { pin -> memcpy(vertexBuffer.contents(), pin.addressOf(0), (data.size * Float.SIZE_BYTES).convert()) }

    println(vertexBuffer.contents())

    // 1
    //val defaultLibrary = device.newDefaultLibrary()!!
    val errorPtr = alloc<ObjCObjectVar<NSError?>>()
    val defaultLibrary = device.newLibraryWithSource("""
        typedef struct {
            packed_float3 position;
            // [[flat]]
        } VertexInput;
        
        typedef struct {
            float4 position [[position]];
        } Varyings;

        vertex Varyings basic_vertex(
            unsigned int vid [[ vertex_id ]],
            const device VertexInput* vertex_array [[buffer(0)]]
        ) {
            Varyings out;
            auto input = vertex_array[vid];
            {
                out.position = float4(input.position, 1.0); 
            }
            return out;
        }
        
        fragment half4 basic_fragment(
            Varyings in [[stage_in]]
        ) { // 1
          return half4(in.position.x / 100.0, in.position.y / 100.0, 1.0, 1.0);              // 2
        }
    """.trimIndent(), null, error = errorPtr.ptr) ?: error("Can't compile Shader: ${errorPtr.value?.localizedDescription}")

// 2
    val pipelineStateDescriptor = MTLRenderPipelineDescriptor()
    pipelineStateDescriptor.vertexFunction = defaultLibrary.newFunctionWithName("basic_vertex")
    pipelineStateDescriptor.fragmentFunction = defaultLibrary.newFunctionWithName("basic_fragment")
    pipelineStateDescriptor.colorAttachments.objectAtIndexedSubscript(0.convert()).pixelFormat = MTLPixelFormatBGRA8Unorm

// 3
    val pipelineState = device.newRenderPipelineStateWithDescriptor(pipelineStateDescriptor, error = null)!!
    val commandQueue = device.newCommandQueue()!!

    val drawable = layer.nextDrawable()
    //println(drawable!!.texture!!.width)
    val renderPassDescriptor = MTLRenderPassDescriptor()
    val colorAttachment = renderPassDescriptor.colorAttachments.objectAtIndexedSubscript(0.convert())
    colorAttachment.texture = drawable!!.texture
    colorAttachment.loadAction = MTLLoadActionClear
    colorAttachment.clearColor = MTLClearColorMake(0.0, 104.0/255.0, 55.0/255.0, 1.0)

    val commandBuffer = commandQueue.commandBuffer()!!

    val renderEncoder = commandBuffer.renderCommandEncoderWithDescriptor(renderPassDescriptor)!!
    renderEncoder.setRenderPipelineState(pipelineState)
    renderEncoder.setVertexBuffer(vertexBuffer, offset = 0, atIndex = 0)
    renderEncoder.drawPrimitives(primitiveType = MTLPrimitiveTypeTriangle, vertexStart = 0, vertexCount = 3, instanceCount = 1)
    renderEncoder.endEncoding()

    commandBuffer.presentDrawable(drawable)
    commandBuffer.commit()
    commandBuffer.waitUntilCompleted()

    val out = ByteArray(width * height * 4)

    out.usePinned { pin ->
        drawable!!.texture!!.getBytes(
            pin.addressOf(0),
            (width * 4).convert(),
            (width * height * 4).convert(),
            MTLRegionMake2D(0.convert(), 0.convert(), width.convert(), height.convert()),
            0.convert(),
            0.convert()
        )
    }

    println("DONE!")

    fun LO(v: Int): kotlin.Byte = ((v ushr 0) and 0xFF).toByte()
    fun HI(v: Int): kotlin.Byte = ((v ushr 8) and 0xFF).toByte()

    out.usePinned { pin ->
        println(out.toList())
        val f = fopen("out.tga", "wb")
        // https://gshaw.ca/closecombat/formats/tga.html
        byteArrayOf(
            0, 0, 2,
            0, 0, 0, 0, 0,

            0, 0, 0, 0,
            LO(width), HI(width), LO(height), HI(height),
            32, 0,
        ).usePinned { header ->
            fwrite(header.addressOf(0), 1, header.get().size.convert(), f)
        }
        //fprintf(f, "P6\n%d %d\n255\n", width, height);
        fwrite(pin.addressOf(0), 1, out.size.convert(), f)
        fclose(f)
    }
}

[soywiz]

Shaders need to know the vertex layouts, and uniforms should be / can be in buffers.

[ygdrasil-io]

Not sure if that can help, but I started to try to convert Apple samples in kotlin some time ago :
https://github.com/ygdrasil-io/Kotlin-native---OSX-Metal-playground

My experience in graphics pipeline is really limited, but if you need some contribution, please ask :)

[soywiz]

Cool! That would be useful.

So the plan is:

There is an AG rewrite in the works.
Now the full backend implementation fits in a single file: https://github.com/korlibs/korge/blob/a2aa6af912a0850b00c8f1f586ea0fda2984f13b/korgw/src/commonMain/kotlin/com/soywiz/korag/gl/AGOpengl.kt

Vertices and vertices layouts are already in a good shape, graphics state also looks good, but the uniforms implementation still requires some work. I want the KorGE batcher to generate several uniform buffers directly with specific layouts (one uniform buffer with the matrices, and other/s with specific values per shader), by generating in a single buffer, batches could reference just to pointers into that buffer, so with that it will be probably in the right order of magnitude in terms of performance.

Also since metal shaders just receive pointers, we could just upload all the uniforms in a single command, and then reference to specific points where the uniforms required for each draw are.

Also metal shaders require vertices and uniforms to be in a specific layout, that is not the case for plain OpenGL.

So the roadmap for this feature is:

Proper AG design:

• Complete the multiple programs/uniforms Support multiple programs/uniforms in BatchBuilder2D #1151 per batch. It is working already with an impressive performance (400 buttons with two shaders per button intercalated at 60fps), but has bugs that need to be fixed. Since it is in the works I want to finish that first.
• Rework how uniforms are sent to AG. We should send a buffer, then just pointers to those buffers, so we can use UBOs on OpenGL, and in metal we won't need to transform the layout of the data since it will be already there. We should consider multiple Uniform blocks (structures/buffers in metal)

Metal implementation (parallelizable):

• AG Program AST to metal shaders. Now there is a GLSL generator from the AST, now we need the same for metal. Here we will need to describe uniform blocks with a specific layout beforehand. Since GLSL shaders don't need the vertex layout beforehand, but metal require it. We could generate different programs based on the program AST + the layout. Or directly attach the vertex layout to the program beforehand always. This will probably be required for Vulkan/WebGPU.
• Actual AG implementation with metal. In the end we can start with something easy, then optimize and figure out ways to take advantage of more features. The actual implementation should be, synchronizing the state: stencil, mask, scissors, viewport, culling, etc., uploading buffers, generating programs and textures from AGObject/AGBuffer/AGTexture, etc. stuff, and then issue commands for rendering. And that's it.

If you want to contribute already, the best place would be AST to metal shaders I guess. Even if the work is partial, that can be started already.
The AST to GLSL (OpenGL one) that can be used as reference is here: https://github.com/korlibs/korge/blob/a2aa6af912a0850b00c8f1f586ea0fda2984f13b/korgw/src/commonMain/kotlin/com/soywiz/korag/shader/gl/GlslGenerator.kt

Once the uniform buffer work is there, the metal implementation could be completed being as fast as possible.

Aditionally. It would be possible already to start the metal backend duplicating the:
https://github.com/korlibs/korge/blob/a2aa6af912a0850b00c8f1f586ea0fda2984f13b/korgw/src/commonMain/kotlin/com/soywiz/korag/gl/AGOpengl.kt
And manually uploading uniforms in each batch, and with fake shaders as a baby step to get it working. As long as we know that later it will require some work to drop some code.

I'm in holidays, so I won't advance pretty much, but I won't probably work on metal directly just yet, and will just work on preparing AG and the batcher to use uniform buffers, to ensure it will work later. I did this proof of concept to be sure the requirements of the AG API to be compatible with Metal while still being fast. But I'm in holidays so I won't code much or at all.

So if you want to start prototyping with AST -> metal shader conversion, or with the betal backend, go ahead and make a PR :)