Running into a panic with AddToScheme

[AtzeDeVries]

What happened?

I am building a function based on this SDK (0.2.0). The functions runs regularly into a panic. This happens somewhere between 1 minutes and 30 minutes after start. The 'other' behaviour of the function is ok.

This is the beginning of the panic.

{"level":"info","ts":1718046487.051641,"caller":"fn/fn.go:41","msg":"Running function","tag":""}
{"level":"info","ts":1718046487.051857,"caller":"fn/fn.go:41","msg":"Running function","tag":""}
fatal error: concurrent map read and map write

goroutine 2882 [running]:
k8s.io/apimachinery/pkg/runtime.(*Scheme).AddKnownTypeWithName(0xc000250000, {{0x0, 0x0}, {0x1a185cf, 0x2}, {0x16df98b, 0x6}}, {0x1cf7bf0, 0xc00044a640})
        /go/pkg/mod/k8s.io/[email protected]/pkg/runtime/scheme.go:170 +0x1d2
k8s.io/apimachinery/pkg/runtime.(*Scheme).AddKnownTypes(0xc000250000, {{0x0?, 0x10?}, {0x1a185cf?, 0xc000617380?}}, {0xc000617580?, 0x5?, 0x1a222b8?})
        /go/pkg/mod/k8s.io/[email protected]/pkg/runtime/scheme.go:148 +0x165
k8s.io/apimachinery/pkg/runtime.(*Scheme).AddUnversionedTypes(0xc000250000, {{0x0?, 0x16?}, {0x1a185cf?, 0xa?}}, {0xc000617580, 0x5, 0x5?})
        /go/pkg/mod/k8s.io/[email protected]/pkg/runtime/scheme.go:124 +0x9c
k8s.io/apimachinery/pkg/apis/meta/v1.AddToGroupVersion(0xc000250000, {{0x1a33ca7?, 0xc000617620?}, {0x1a1f4ca?, 0x6?}})
        /go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/register.go:75 +0x345
github.com/coopnorge/provider-github/apis/repo/v1alpha1.init.0.(*Builder).Register.func1(0xc000250000)
        /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/scheme/scheme.go:72 +0x5a
k8s.io/apimachinery/pkg/runtime.(*SchemeBuilder).AddToScheme(...)
        /go/pkg/mod/k8s.io/[email protected]/pkg/runtime/scheme_builder.go:29
sigs.k8s.io/controller-runtime/pkg/scheme.(*Builder).AddToScheme(0x1877e20?, 0xc000250000)
        /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/scheme/scheme.go:86 +0x56
main.(*Function).RunFunction(0xc0001fef50, {0x1824660?, 0xc0008139a0?}, 0xc0000d2600)
        /fn/fn.go:79 +0x42f
github.com/crossplane/function-sdk-go/proto/v1beta1._FunctionRunnerService_RunFunction_Handler({0x1824660, 0xc0001fef50}, {0x1d0a6c0, 0xc00096f680}, 0xc0004e2300, 0x0)
        /go/pkg/mod/github.com/crossplane/[email protected]/proto/v1beta1/run_function_grpc.pb.go:104 +0x1a6
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0000ef800, {0x1d0a6c0, 0xc00096f5c0}, {0x1d12620, 0xc000686340}, 0xc00056f8c0, 0xc00043ca80, 0x2a3acc0, 0x0)
        /go/pkg/mod/google.golang.org/[email protected]/server.go:1385 +0xdd1
google.golang.org/grpc.(*Server).handleStream(0xc0000ef800, {0x1d12620, 0xc000686340}, 0xc00056f8c0)
        /go/pkg/mod/google.golang.org/[email protected]/server.go:1796 +0xfb8
google.golang.org/grpc.(*Server).serveStreams.func2.1()
        /go/pkg/mod/google.golang.org/[email protected]/server.go:1029 +0x8b
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 11
        /go/pkg/mod/google.golang.org/[email protected]/server.go:1040 +0x125

goroutine 1 [IO wait, 9 minutes]:
internal/poll.runtime_pollWait(0x7f941cd43e70, 0x72)
        /usr/local/go/src/runtime/netpoll.go:345 +0x85
internal/poll.(*pollDesc).wait(0x3?, 0x4271e5?, 0x0)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
        /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0005f6180)
        /usr/local/go/src/internal/poll/fd_unix.go:611 +0x2ac
net.(*netFD).accept(0xc0005f6180)
        /usr/local/go/src/net/fd_unix.go:172 +0x29
net.(*TCPListener).accept(0xc000430280)
        /usr/local/go/src/net/tcpsock_posix.go:159 +0x1e
net.(*TCPListener).Accept(0xc000430280)
        /usr/local/go/src/net/tcpsock.go:327 +0x30
google.golang.org/grpc.(*Server).Serve(0xc0000ef800, {0x1d01778, 0xc000430280})
        /go/pkg/mod/google.golang.org/[email protected]/server.go:885 +0x469
github.com/crossplane/function-sdk-go.Serve({0x1cf78a8, 0xc0001fef50}, {0xc000439420, 0x3, 0xc0004dfa40?})
        /go/pkg/mod/github.com/crossplane/[email protected]/sdk.go:143 +0x225
main.(*CLI).Run(0xc0004c0540)
        /fn/main.go:27 +0x196
reflect.Value.call({0x17f6940?, 0xc0004c0540?, 0x419f45?}, {0x1a192d6, 0x4}, {0xc00050fb90, 0x0, 0x16d53b9?})
        /usr/local/go/src/reflect/value.go:596 +0xca6
reflect.Value.Call({0x17f6940?, 0xc0004c0540?, 0x182c5e0?}, {0xc00050fb90?, 0xc00050fbd0?, 0x411b9b?})
        /usr/local/go/src/reflect/value.go:380 +0xb9
github.com/alecthomas/kong.callFunction({0x17f6940?, 0xc0004c0540?, 0xc00050fd88?}, 0xc0004df9b0)
        /go/pkg/mod/github.com/alecthomas/[email protected]/callbacks.go:98 +0x45a
github.com/alecthomas/kong.(*Context).RunNode(0xc0004e2500, 0xc0002bf680, {0x0, 0x0, 0x3?})
        /go/pkg/mod/github.com/alecthomas/[email protected]/context.go:774 +0x80e
github.com/alecthomas/kong.(*Context).Run(0xc0004e2500, {0x0, 0x0, 0x0})
        /go/pkg/mod/github.com/alecthomas/[email protected]/context.go:793 +0xfd
main.main()
        /fn/main.go:35 +0xce

a bit further in the panic

goroutine 2809 [runnable]:
k8s.io/apimachinery/pkg/apis/meta/v1.AddToGroupVersion(0xc000250000, {{0x1a33ca7?, 0xc00053d620?}, {0x1a1f4ca?, 0x6?}})
        /go/pkg/mod/k8s.io/[email protected]/pkg/apis/meta/v1/register.go:79 +0x22d
github.com/coopnorge/provider-github/apis/repo/v1alpha1.init.4.(*Builder).Register.func1(0xc000250000)
        /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/scheme/scheme.go:72 +0x5a
k8s.io/apimachinery/pkg/runtime.(*SchemeBuilder).AddToScheme(...)
        /go/pkg/mod/k8s.io/[email protected]/pkg/runtime/scheme_builder.go:29
sigs.k8s.io/controller-runtime/pkg/scheme.(*Builder).AddToScheme(0x1877e20?, 0xc000250000)
        /go/pkg/mod/sigs.k8s.io/[email protected]/pkg/scheme/scheme.go:86 +0x56
main.(*Function).RunFunction(0xc0001fef50, {0x1824660?, 0xc00085e9a0?}, 0xc000144480)
        /fn/fn.go:79 +0x42f
github.com/crossplane/function-sdk-go/proto/v1beta1._FunctionRunnerService_RunFunction_Handler({0x1824660, 0xc0001fef50}, {0x1d0a6c0, 0xc000764240}, 0xc00037a200, 0x0)
        /go/pkg/mod/github.com/crossplane/[email protected]/proto/v1beta1/run_function_grpc.pb.go:104 +0x1a6
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0000ef800, {0x1d0a6c0, 0xc000764180}, {0x1d12620, 0xc000686340}, 0xc0004057a0, 0xc00043ca80, 0x2a3acc0, 0x0)
        /go/pkg/mod/google.golang.org/[email protected]/server.go:1385 +0xdd1
google.golang.org/grpc.(*Server).handleStream(0xc0000ef800, {0x1d12620, 0xc000686340}, 0xc0004057a0)
        /go/pkg/mod/google.golang.org/[email protected]/server.go:1796 +0xfb8
google.golang.org/grpc.(*Server).serveStreams.func2.1()
        /go/pkg/mod/google.golang.org/[email protected]/server.go:1029 +0x8b
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 11
        /go/pkg/mod/google.golang.org/[email protected]/server.go:1040 +0x125

On line 79 in fn.go the AddToScheme happens:

err = repo.AddToScheme(composed.Scheme)
if err != nil {
        response.Fatal(rsp, errors.Wrapf(err, "Unable to add v1alpha1/repo to scheme %T", req))
	return rsp, nil
 }

And repo is imported as

	repo "github.com/coopnorge/provider-github/apis/repo/v1alpha1"

I am not sure where to go from here.

How can we reproduce it?

I dont know, help is welcome.

What environment did it happen in?

Crossplane version: 1.15.3

Env: GKE 1.27