Skip to content

Commit

Permalink
Fix OciRepositoryHandler.getOrHeadMetadata:
Browse files Browse the repository at this point in the history 
The image metadata can be different (missing indexAnnotations and manifestDescriptorAnnotations) depending on if it is served from cache or not, because the digest/size of the single image manifest is used.
Instead, the digest that is referenced by the imageReference (could be an index) must be used.
This requires to add the platform as a path parameter to the oci metadata endpoint to select the right metadata.
  • Loading branch information
@SgtSilvio
SgtSilvio committed Jul 13, 2024
1 parent 5d4709b commit bd3bc37
Show file tree
Hide file tree
Showing 2 changed files with 75 additions and 66 deletions.
70 changes: 37 additions & 33 deletions src/main/kotlin/io/github/sgtsilvio/gradle/oci/internal/registry/OciMetadataRegistry.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,40 +13,40 @@ import java.util.*
*/
internal class OciMetadataRegistry(val registryApi: OciRegistryApi) {

data class Metadata(val metadata: OciMetadata, val platform: Platform, val digest: OciDigest, val size: Int)
class OciImageMetadata(val platformToMetadata: Map<Platform, OciMetadata>, val digest: OciDigest, val size: Int)

fun pullMetadataList(
fun pullImageMetadata(
registry: String,
imageReference: OciImageReference,
credentials: Credentials?,
): Mono<List<Metadata>> =
): Mono<OciImageMetadata> =
registryApi.pullManifest(registry, imageReference.name, imageReference.tag.replaceFirst('!', ':'), credentials)
.transformToMetadataList(registry, imageReference, credentials)
.transformToImageMetadata(registry, imageReference, credentials)

fun pullMetadataList(
fun pullImageMetadata(
registry: String,
imageReference: OciImageReference,
digest: OciDigest,
size: Int,
credentials: Credentials?,
): Mono<List<Metadata>> = registryApi.pullManifest(registry, imageReference.name, digest, size, credentials)
.transformToMetadataList(registry, imageReference, credentials)
): Mono<OciImageMetadata> = registryApi.pullManifest(registry, imageReference.name, digest, size, credentials)
.transformToImageMetadata(registry, imageReference, credentials)

private fun Mono<OciData>.transformToMetadataList(
private fun Mono<OciData>.transformToImageMetadata(
registry: String,
imageReference: OciImageReference,
credentials: Credentials?,
): Mono<List<Metadata>> = flatMap { manifest ->
transformToMetadataList(registry, imageReference, manifest, credentials)
): Mono<OciImageMetadata> = flatMap { manifest ->
transformToImageMetadata(registry, imageReference, manifest, credentials)
}

private fun transformToMetadataList(
private fun transformToImageMetadata(
registry: String,
imageReference: OciImageReference,
manifest: OciData,
credentials: Credentials?,
): Mono<List<Metadata>> = when (manifest.mediaType) {
INDEX_MEDIA_TYPE -> transformIndexToMetadataList(
): Mono<OciImageMetadata> = when (manifest.mediaType) {
INDEX_MEDIA_TYPE -> transformIndexToImageMetadata(
registry,
imageReference,
manifest,
Expand All @@ -56,7 +56,7 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) {
LAYER_MEDIA_TYPE_PREFIX,
)

MANIFEST_MEDIA_TYPE -> transformManifestToMetadataList(
MANIFEST_MEDIA_TYPE -> transformManifestToImageMetadata(
registry,
imageReference,
manifest,
Expand All @@ -65,7 +65,7 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) {
LAYER_MEDIA_TYPE_PREFIX,
)

DOCKER_MANIFEST_LIST_MEDIA_TYPE -> transformIndexToMetadataList(
DOCKER_MANIFEST_LIST_MEDIA_TYPE -> transformIndexToImageMetadata(
registry,
imageReference,
manifest,
Expand All @@ -75,7 +75,7 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) {
DOCKER_LAYER_MEDIA_TYPE,
)

DOCKER_MANIFEST_MEDIA_TYPE -> transformManifestToMetadataList(
DOCKER_MANIFEST_MEDIA_TYPE -> transformManifestToImageMetadata(
registry,
imageReference,
manifest,
Expand All @@ -87,20 +87,20 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) {
else -> throw IllegalStateException("unsupported manifest media type '${manifest.mediaType}'")
}

private fun transformIndexToMetadataList(
private fun transformIndexToImageMetadata(
registry: String,
imageReference: OciImageReference,
index: OciData,
credentials: Credentials?,
manifestMediaType: String,
configMediaType: String,
layerMediaTypePrefix: String,
): Mono<List<Metadata>> {
): Mono<OciImageMetadata> {
val indexJsonObject = jsonObject(String(index.bytes))
val indexAnnotations = indexJsonObject.getStringMapOrEmpty("annotations")
val metadataMonoList = indexJsonObject.get("manifests") {
asArray().toList {
val (platform, manifestDescriptor) = asObject().decodeOciManifestDescriptor()
val (manifestDescriptorPlatform, manifestDescriptor) = asObject().decodeOciManifestDescriptor()
if (manifestDescriptor.mediaType != manifestMediaType) { // TODO support nested index
Mono.empty()
} else {
Expand All @@ -112,7 +112,7 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) {
credentials,
).flatMap { manifest ->
if (manifest.mediaType != manifestMediaType) {
throw IllegalArgumentException("media type in manifest descriptor ($manifestMediaType) and manifest (${manifest.mediaType}) do not match")
throw IllegalStateException("media type in manifest descriptor ($manifestMediaType) and manifest (${manifest.mediaType}) do not match")
}
transformManifestToMetadata(
registry,
Expand All @@ -124,29 +124,35 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) {
configMediaType,
layerMediaTypePrefix,
)
}.map { metadata ->
if ((platform != null) && (metadata.platform != platform)) {
throw IllegalArgumentException("platform in manifest descriptor ($platform) and config (${metadata.platform}) do not match")
}.doOnNext { (platform) ->
if ((manifestDescriptorPlatform != null) && (platform != manifestDescriptorPlatform)) {
throw IllegalStateException("platform in manifest descriptor ($manifestDescriptorPlatform) and config ($platform) do not match")
}
metadata
}
}
}
}
indexJsonObject.requireStringOrNull("mediaType", index.mediaType)
indexJsonObject.requireLong("schemaVersion", 2)
// the same order as in the manifest is guaranteed by mergeSequential
return Flux.mergeSequential(metadataMonoList).collectList()
return Flux.mergeSequential(metadataMonoList)
// linked to preserve the platform order
.collect({ LinkedHashMap<Platform, OciMetadata>() }) { map, (platform, metadata) ->
if (map.putIfAbsent(platform, metadata) != null) {
throw IllegalStateException("duplicate platform in image index: $platform")
}
}
.map { OciImageMetadata(it, index.digest, index.bytes.size) }
}

private fun transformManifestToMetadataList(
private fun transformManifestToImageMetadata(
registry: String,
imageReference: OciImageReference,
manifest: OciData,
credentials: Credentials?,
configMediaType: String,
layerMediaTypePrefix: String,
): Mono<List<Metadata>> = transformManifestToMetadata(
): Mono<OciImageMetadata> = transformManifestToMetadata(
registry,
imageReference,
manifest,
Expand All @@ -155,7 +161,7 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) {
credentials,
configMediaType,
layerMediaTypePrefix,
).map { listOf(it) }
).map { OciImageMetadata(mapOf(it), manifest.digest, manifest.bytes.size) }

private fun transformManifestToMetadata(
registry: String,
Expand All @@ -166,7 +172,7 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) {
credentials: Credentials?,
configMediaType: String,
layerMediaTypePrefix: String,
): Mono<Metadata> {
): Mono<Pair<Platform, OciMetadata>> {
val manifestJsonObject = jsonObject(String(manifest.bytes))
val manifestAnnotations = manifestJsonObject.getStringMapOrEmpty("annotations")
val configDescriptor = manifestJsonObject.get("config") { asObject().decodeOciDescriptor() }
Expand Down Expand Up @@ -272,7 +278,8 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) {
)
}

Metadata(
Pair(
Platform(os, architecture, variant, osVersion, osFeatures),
OciMetadata(
imageReference,
creationTime,
Expand All @@ -292,9 +299,6 @@ internal class OciMetadataRegistry(val registryApi: OciRegistryApi) {
indexAnnotations,
layers,
),
Platform(os, architecture, variant, osVersion, osFeatures),
manifest.digest,
manifest.bytes.size,
)
}
}
Expand Down
71 changes: 38 additions & 33 deletions src/main/kotlin/io/github/sgtsilvio/gradle/oci/internal/registry/OciRepositoryHandler.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import io.github.sgtsilvio.gradle.oci.mapping.VersionedCoordinates
import io.github.sgtsilvio.gradle.oci.mapping.map
import io.github.sgtsilvio.gradle.oci.metadata.*
import io.github.sgtsilvio.gradle.oci.platform.Platform
import io.github.sgtsilvio.gradle.oci.platform.toPlatform
import io.netty.handler.codec.http.HttpHeaderNames
import io.netty.handler.codec.http.HttpHeaderValues
import io.netty.handler.codec.http.HttpMethod
Expand All @@ -38,7 +39,7 @@ import java.util.function.BiFunction
/v2/repository/<base64(registryUrl)> / <group>/<name>/<version> / <variantName>/<digest>/<size>/<...>oci-layer
/v0.11/<escapeSlash(registryUrl)> / <group>/<name>/<version> / <...>.module
/v0.11/<escapeSlash(registryUrl)> / <group>/<name>/<version> / <escapeSlash(imageReference)>/<digest>/<size>/<...>.json
/v0.11/<escapeSlash(registryUrl)> / <group>/<name>/<version> / <escapeSlash(imageReference)>/<digest>/<size>/<platform>/<...>.json
/v0.11/<escapeSlash(registryUrl)> / <group>/<name>/<version> / <escapeSlash(imageName)>/<digest>/<size>/<...>oci-layer
*/

Expand All @@ -51,10 +52,10 @@ internal class OciRepositoryHandler(
private val credentials: Credentials?,
) : BiFunction<HttpServerRequest, HttpServerResponse, Publisher<Void>> {

private val metadataCache: AsyncCache<ComponentCacheKey, List<OciMetadataRegistry.Metadata>> =
private val imageMetadataCache: AsyncCache<ImageMetadataCacheKey, OciMetadataRegistry.OciImageMetadata> =
Caffeine.newBuilder().maximumSize(100).expireAfterAccess(1, TimeUnit.MINUTES).buildAsync()

private data class ComponentCacheKey(
private data class ImageMetadataCacheKey(
val registry: String,
val imageReference: OciImageReference,
val digest: OciDigest?,
Expand Down Expand Up @@ -109,7 +110,7 @@ internal class OciRepositoryHandler(
isGet: Boolean,
response: HttpServerResponse,
): Publisher<Void> {
if (segments.size != 8) {
if (segments.size != 9) {
return response.sendNotFound()
}
val imageReference = try {
Expand All @@ -127,9 +128,20 @@ internal class OciRepositoryHandler(
} catch (e: NumberFormatException) {
return response.sendBadRequest()
}
val metadataJsonMono = getMetadata(registryUri, imageReference, digest, size, credentials).map { (metadata) ->
metadata.encodeToJsonString().toByteArray()
val platform = try {
segments[7].toPlatform()
} catch (e: IllegalArgumentException) {
return response.sendBadRequest()
}
val metadataJsonMono =
getImageMetadata(registryUri, imageReference, digest, size, credentials).handle { imageMetadata, sink ->
val metadata = imageMetadata.platformToMetadata[platform]
if (metadata == null) {
response.status(400)
} else {
sink.next(metadata.encodeToJsonString().toByteArray())
}
}
response.header(HttpHeaderNames.CONTENT_TYPE, HttpHeaderValues.APPLICATION_JSON)
return response.sendByteArray(metadataJsonMono, isGet)
}
Expand Down Expand Up @@ -176,8 +188,8 @@ internal class OciRepositoryHandler(
): Publisher<Void> {
val componentId = mappedComponent.componentId
val variantMetadataMonoList = mappedComponent.variants.map { (variantName, variant) ->
getMetadataList(registryUri, variant.imageReference, credentials).map { metadataList ->
Triple(variantName, variant.capabilities, metadataList)
getImageMetadata(registryUri, variant.imageReference, credentials).map { imageMetadata ->
Triple(variantName, variant.capabilities, imageMetadata)
}
}
val moduleJsonMono = variantMetadataMonoList.zip { variantMetadataList ->
Expand All @@ -193,18 +205,18 @@ internal class OciRepositoryHandler(
}
val fileNamePrefix = "${componentId.name}-${componentId.version}-"
addArray("variants") {
for ((variantName, capabilities, metadataList) in variantMetadataList) {
for ((variantName, capabilities, imageMetadata) in variantMetadataList) {
addObject {
addString("name", createOciVariantName(variantName))
addOciVariantAttributes(MULTIPLE_PLATFORMS_ATTRIBUTE_VALUE)
addCapabilities("capabilities", capabilities, componentId)
addArray("dependencies") {
for ((_, platform) in metadataList) {
for (platform in imageMetadata.platformToMetadata.keys) {
addDependency(componentId, capabilities, platform)
}
}
}
for ((metadata, platform, digest, size) in metadataList) {
for ((platform, metadata) in imageMetadata.platformToMetadata) {
addObject {
addString("name", createOciVariantName(variantName, platform))
addOciVariantAttributes(platform.toString())
Expand All @@ -223,7 +235,7 @@ internal class OciRepositoryHandler(
val metadataName = fileNamePrefix + createOciMetadataClassifier(variantName) + createPlatformPostfix(platform) + ".json"
val escapedImageReference = metadata.imageReference.toString().escapePathSegment()
addString("name", metadataName)
addString("url", "$escapedImageReference/$digest/$size/$metadataName")
addString("url", "$escapedImageReference/${imageMetadata.digest}/${imageMetadata.size}/$platform/$metadataName")
addNumber("size", metadataJson.size.toLong())
addString("sha512", DigestUtils.sha512Hex(metadataJson))
addString("sha256", DigestUtils.sha256Hex(metadataJson))
Expand Down Expand Up @@ -256,41 +268,34 @@ internal class OciRepositoryHandler(
return response.sendByteArray(moduleJsonMono, isGet)
}

private fun getMetadataList(
private fun getImageMetadata(
registryUri: URI,
imageReference: OciImageReference,
credentials: Credentials?,
): Mono<List<OciMetadataRegistry.Metadata>> {
return metadataCache.getMono(
ComponentCacheKey(registryUri.toString(), imageReference, null, -1, credentials?.hashed())
): Mono<OciMetadataRegistry.OciImageMetadata> {
return imageMetadataCache.getMono(
ImageMetadataCacheKey(registryUri.toString(), imageReference, null, -1, credentials?.hashed())
) { key ->
metadataRegistry.pullMetadataList(key.registry, key.imageReference, credentials).doOnNext { metadataList ->
for (metadata in metadataList) {
metadataCache.asMap().putIfAbsent(
key.copy(digest = metadata.digest, size = metadata.size),
CompletableFuture.completedFuture(listOf(metadata)),
)
}
metadataRegistry.pullImageMetadata(key.registry, key.imageReference, credentials).doOnNext {
imageMetadataCache.asMap().putIfAbsent(
key.copy(digest = it.digest, size = it.size),
CompletableFuture.completedFuture(it),
)
}
}
}

private fun getMetadata(
private fun getImageMetadata(
registryUri: URI,
imageReference: OciImageReference,
digest: OciDigest,
size: Int,
credentials: Credentials?,
): Mono<OciMetadataRegistry.Metadata> {
return metadataCache.getMono(
ComponentCacheKey(registryUri.toString(), imageReference, digest, size, credentials?.hashed())
): Mono<OciMetadataRegistry.OciImageMetadata> {
return imageMetadataCache.getMono(
ImageMetadataCacheKey(registryUri.toString(), imageReference, digest, size, credentials?.hashed())
) { (registry, imageReference) ->
metadataRegistry.pullMetadataList(registry, imageReference, digest, size, credentials)
}.map { metadataList ->
if (metadataList.size != 1) {
throw IllegalStateException() // TODO message
}
metadataList[0]
metadataRegistry.pullImageMetadata(registry, imageReference, digest, size, credentials)
}
}

Expand Down

0 comments on commit bd3bc37

Please sign in to comment.