diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/ncache/Cache.java b/jib-core/src/main/java/com/google/cloud/tools/jib/ncache/Cache.java index 2a1177aa29..80eeeacdbe 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/ncache/Cache.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/ncache/Cache.java @@ -23,14 +23,15 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.attribute.FileTime; import java.util.Optional; import javax.annotation.concurrent.Immutable; /** * Cache for storing data to be shared between Jib executions. * - *

Uses the default cache storage engine ({@link DefaultCacheStorage}) and layer entries as the - * selector ({@link LayerEntriesSelector}). + *

Uses the default cache storage engine ({@link DefaultCacheStorage}), layer entries as the + * selector ({@link LayerEntriesSelector}), and last modified time as the metadata. * *

This class is immutable and safe to use across threads. */ @@ -56,8 +57,8 @@ private Cache(CacheStorage cacheStorage) { } /** - * Saves a cache entry with only a layer {@link Blob}. Use {@link #write(Blob, ImmutableList, - * Blob)} to include a selector and metadata. + * Saves a cache entry with only a layer {@link Blob}. Use {@link #write(Blob, ImmutableList)} to + * include a selector and metadata. * * @param layerBlob the layer {@link Blob} * @return the {@link CacheEntry} for the written layer @@ -73,19 +74,22 @@ public CacheEntry write(Blob layerBlob) throws IOException { * * @param layerBlob the layer {@link Blob} * @param layerEntries the layer entries that make up the layer - * @param metadataBlob the metadata {@link Blob} * @return the {@link CacheEntry} for the written layer and metadata * @throws IOException if an I/O exception occurs */ - public CacheEntry write(Blob layerBlob, ImmutableList layerEntries, Blob metadataBlob) + public CacheEntry write(Blob layerBlob, ImmutableList layerEntries) throws IOException { return cacheStorage.write( CacheWrite.withSelectorAndMetadata( - layerBlob, LayerEntriesSelector.generateSelector(layerEntries), metadataBlob)); + layerBlob, + LayerEntriesSelector.generateSelector(layerEntries), + LastModifiedTimeMetadata.generateMetadata(layerEntries))); } /** - * Retrieves the {@link CacheEntry} that was built from the {@code layerEntries}. + * Retrieves the {@link CacheEntry} that was built from the {@code layerEntries}. The last + * modified time of the {@code layerEntries} must match the last modified time as stored by the + * metadata of the {@link CacheEntry}. * * @param layerEntries the layer entries to match against * @return a {@link CacheEntry} that was built from {@code layerEntries}, if found @@ -100,7 +104,28 @@ public Optional retrieve(ImmutableList layerEntries) return Optional.empty(); } - return cacheStorage.retrieve(optionalSelectedLayerDigest.get()); + Optional optionalCacheEntry = + cacheStorage.retrieve(optionalSelectedLayerDigest.get()); + if (!optionalCacheEntry.isPresent()) { + return Optional.empty(); + } + + CacheEntry cacheEntry = optionalCacheEntry.get(); + + Optional optionalRetrievedLastModifiedTime = + LastModifiedTimeMetadata.getLastModifiedTime(cacheEntry); + if (!optionalRetrievedLastModifiedTime.isPresent()) { + return Optional.empty(); + } + + FileTime retrievedLastModifiedTime = optionalRetrievedLastModifiedTime.get(); + FileTime expectedLastModifiedTime = LastModifiedTimeMetadata.getLastModifiedTime(layerEntries); + + if (!expectedLastModifiedTime.equals(retrievedLastModifiedTime)) { + return Optional.empty(); + } + + return Optional.of(cacheEntry); } /** diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/ncache/LastModifiedTimeMetadata.java b/jib-core/src/main/java/com/google/cloud/tools/jib/ncache/LastModifiedTimeMetadata.java new file mode 100644 index 0000000000..6204bc11fb --- /dev/null +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/ncache/LastModifiedTimeMetadata.java @@ -0,0 +1,88 @@ +/* + * Copyright 2018 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.ncache; + +import com.google.cloud.tools.jib.blob.Blob; +import com.google.cloud.tools.jib.blob.Blobs; +import com.google.cloud.tools.jib.image.LayerEntry; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.attribute.FileTime; +import java.time.Instant; +import java.util.Optional; + +/** + * Serializes/deserializes metadata storing the latest last modified time of all the source files in + * {@link LayerEntry}s for a layer. + * + *

Use {@link #generateMetadata} to serialize the latest last modified time of all the source + * files in {@link LayerEntry}s for a layer into a {@link Blob} containing the serialized last + * modified time. Use {@link #getLastModifiedTime(CacheEntry)} to deserialize the metadata in a + * {@link CacheEntry} into a last modified time. + */ +class LastModifiedTimeMetadata { + + /** + * Generates the metadata {@link Blob} for the list of {@link LayerEntry}s. The metadata is the + * latest last modified time of all the source files in the list of {@link LayerEntry}s serialized + * using {@link Instant#toString}. + * + * @param layerEntries the list of {@link LayerEntry}s + * @return the generated metadata + */ + static Blob generateMetadata(ImmutableList layerEntries) throws IOException { + return Blobs.from(getLastModifiedTime(layerEntries).toInstant().toString()); + } + + /** + * Gets the latest last modified time of all the source files in the list of {@link LayerEntry}s. + * + * @param layerEntries the list of {@link LayerEntry}s + * @return the last modified time + */ + static FileTime getLastModifiedTime(ImmutableList layerEntries) throws IOException { + FileTime maxLastModifiedTime = FileTime.from(Instant.MIN); + + for (LayerEntry layerEntry : layerEntries) { + FileTime lastModifiedTime = Files.getLastModifiedTime(layerEntry.getSourceFile()); + if (lastModifiedTime.compareTo(maxLastModifiedTime) > 0) { + maxLastModifiedTime = lastModifiedTime; + } + } + + return maxLastModifiedTime; + } + + /** + * Gets the last modified time from the metadata of a {@link CacheEntry}. + * + * @param cacheEntry the {@link CacheEntry} + * @return the last modified time, if the metadata is present + * @throws IOException if deserialization of the metadata failed + */ + static Optional getLastModifiedTime(CacheEntry cacheEntry) throws IOException { + if (!cacheEntry.getMetadataBlob().isPresent()) { + return Optional.empty(); + } + + Blob metadataBlob = cacheEntry.getMetadataBlob().get(); + return Optional.of(FileTime.from(Instant.parse(Blobs.writeToString(metadataBlob)))); + } + + private LastModifiedTimeMetadata() {} +} diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/ncache/CacheTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/ncache/CacheTest.java index 04b2344fea..31e901d4bb 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/ncache/CacheTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/ncache/CacheTest.java @@ -27,8 +27,10 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; +import java.nio.file.attribute.FileTime; +import java.time.Instant; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import org.junit.Assert; @@ -104,35 +106,38 @@ private static long sizeOf(Blob blob) throws IOException { private DescriptorDigest layerDiffId1; private long layerSize1; private ImmutableList layerEntries1; - private Blob metadataBlob1; private Blob layerBlob2; private DescriptorDigest layerDigest2; private DescriptorDigest layerDiffId2; private long layerSize2; private ImmutableList layerEntries2; - private Blob metadataBlob2; @Before public void setUp() throws IOException { + Path directory = temporaryFolder.newFolder().toPath(); + Files.createDirectory(directory.resolve("source")); + Files.createFile(directory.resolve("source/file")); + Files.createDirectories(directory.resolve("another/source")); + Files.createFile(directory.resolve("another/source/file")); + layerBlob1 = Blobs.from("layerBlob1"); layerDigest1 = digestOf(compress(layerBlob1)); layerDiffId1 = digestOf(layerBlob1); layerSize1 = sizeOf(compress(layerBlob1)); layerEntries1 = ImmutableList.of( - new LayerEntry(Paths.get("source/file"), AbsoluteUnixPath.get("/extraction/path")), new LayerEntry( - Paths.get("another/source/file"), + directory.resolve("source/file"), AbsoluteUnixPath.get("/extraction/path")), + new LayerEntry( + directory.resolve("another/source/file"), AbsoluteUnixPath.get("/another/extraction/path"))); - metadataBlob1 = Blobs.from("metadata"); layerBlob2 = Blobs.from("layerBlob2"); layerDigest2 = digestOf(compress(layerBlob2)); layerDiffId2 = digestOf(layerBlob2); layerSize2 = sizeOf(compress(layerBlob2)); layerEntries2 = ImmutableList.of(); - metadataBlob2 = Blobs.from("metadata"); } @Test @@ -159,31 +164,36 @@ public void testWriteLayerOnly_retrieveByLayerDigest() } @Test - public void testWriteWithSelectorAndMetadata_retrieveByLayerDigest() + public void testWriteWithLayerEntries_retrieveByLayerDigest() throws IOException, CacheCorruptedException { Cache cache = Cache.withDirectory(temporaryFolder.newFolder().toPath()); - verifyIsLayer1WithMetadata(cache.write(layerBlob1, layerEntries1, metadataBlob1)); + verifyIsLayer1WithMetadata(cache.write(layerBlob1, layerEntries1)); verifyIsLayer1WithMetadata(cache.retrieve(layerDigest1).orElseThrow(AssertionError::new)); Assert.assertFalse(cache.retrieve(layerDigest2).isPresent()); } @Test - public void testWriteWithSelectorAndMetadata_retrieveByLayerEntries() + public void testWriteWithLayerEntries_retrieveByLayerEntries() throws IOException, CacheCorruptedException { Cache cache = Cache.withDirectory(temporaryFolder.newFolder().toPath()); - verifyIsLayer1WithMetadata(cache.write(layerBlob1, layerEntries1, metadataBlob1)); + verifyIsLayer1WithMetadata(cache.write(layerBlob1, layerEntries1)); verifyIsLayer1WithMetadata(cache.retrieve(layerEntries1).orElseThrow(AssertionError::new)); Assert.assertFalse(cache.retrieve(layerDigest2).isPresent()); + + // A source file modification results in the cached layer to be out-of-date and not retrieved. + Files.setLastModifiedTime( + layerEntries1.get(0).getSourceFile(), FileTime.from(Instant.now().plusSeconds(1))); + Assert.assertFalse(cache.retrieve(layerEntries1).isPresent()); } @Test public void testRetrieveWithTwoEntriesInCache() throws IOException, CacheCorruptedException { Cache cache = Cache.withDirectory(temporaryFolder.newFolder().toPath()); - verifyIsLayer1WithMetadata(cache.write(layerBlob1, layerEntries1, metadataBlob1)); - verifyIsLayer2WithMetadata(cache.write(layerBlob2, layerEntries2, metadataBlob2)); + verifyIsLayer1WithMetadata(cache.write(layerBlob1, layerEntries1)); + verifyIsLayer2WithMetadata(cache.write(layerBlob2, layerEntries2)); verifyIsLayer1WithMetadata(cache.retrieve(layerDigest1).orElseThrow(AssertionError::new)); verifyIsLayer2WithMetadata(cache.retrieve(layerDigest2).orElseThrow(AssertionError::new)); verifyIsLayer1WithMetadata(cache.retrieve(layerEntries1).orElseThrow(AssertionError::new)); @@ -212,7 +222,9 @@ private void verifyIsLayer1NoMetadata(CacheEntry cacheEntry) throws IOException private void verifyIsLayer1WithMetadata(CacheEntry cacheEntry) throws IOException { verifyIsLayer1(cacheEntry); Assert.assertTrue(cacheEntry.getMetadataBlob().isPresent()); - Assert.assertEquals("metadata", Blobs.writeToString(cacheEntry.getMetadataBlob().get())); + Assert.assertEquals( + Blobs.writeToString(LastModifiedTimeMetadata.generateMetadata(layerEntries1)), + Blobs.writeToString(cacheEntry.getMetadataBlob().get())); } /** @@ -242,6 +254,8 @@ private void verifyIsLayer2WithMetadata(CacheEntry cacheEntry) throws IOExceptio Assert.assertEquals(layerDiffId2, cacheEntry.getLayerDiffId()); Assert.assertEquals(layerSize2, cacheEntry.getLayerSize()); Assert.assertTrue(cacheEntry.getMetadataBlob().isPresent()); - Assert.assertEquals("metadata", Blobs.writeToString(cacheEntry.getMetadataBlob().get())); + Assert.assertEquals( + Blobs.writeToString(LastModifiedTimeMetadata.generateMetadata(layerEntries2)), + Blobs.writeToString(cacheEntry.getMetadataBlob().get())); } } diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/ncache/LastModifiedTimeMetadataTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/ncache/LastModifiedTimeMetadataTest.java new file mode 100644 index 0000000000..7d73dce04f --- /dev/null +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/ncache/LastModifiedTimeMetadataTest.java @@ -0,0 +1,126 @@ +/* + * Copyright 2018 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.ncache; + +import com.google.cloud.tools.jib.blob.Blobs; +import com.google.cloud.tools.jib.filesystem.AbsoluteUnixPath; +import com.google.cloud.tools.jib.image.DescriptorDigest; +import com.google.cloud.tools.jib.image.LayerEntry; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Resources; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileTime; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; + +/** Tests for {@link LastModifiedTimeMetadata}. */ +public class LastModifiedTimeMetadataTest { + + private static LayerEntry copyFile(Path source, Path destination, FileTime newLastModifiedTime) + throws IOException { + Files.createDirectories(destination.getParent()); + Files.copy(source, destination); + Files.setLastModifiedTime(destination, newLastModifiedTime); + return new LayerEntry(destination, AbsoluteUnixPath.get("/ignored")); + } + + @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private List layerEntries = new ArrayList<>(); + + @Before + public void setUp() throws IOException, URISyntaxException { + Path originalDirectory = Paths.get(Resources.getResource("layer").toURI()); + Path directory = temporaryFolder.newFolder().toPath(); + + layerEntries.add( + copyFile( + originalDirectory.resolve("a/b/bar"), + directory.resolve("a/b/bar"), + FileTime.fromMillis(1000))); + layerEntries.add( + copyFile( + originalDirectory.resolve("c/cat"), + directory.resolve("c/cat"), + FileTime.fromMillis(2000))); + layerEntries.add( + copyFile( + originalDirectory.resolve("foo"), directory.resolve("foo"), FileTime.fromMillis(1500))); + } + + @Test + public void testGetLastModifiedTime_layerEntries() throws IOException { + Assert.assertEquals( + FileTime.fromMillis(2000), + LastModifiedTimeMetadata.getLastModifiedTime(ImmutableList.copyOf(layerEntries))); + } + + @Test + public void testGetLastModifiedTime_noEntries() throws IOException { + Assert.assertEquals( + FileTime.from(Instant.MIN), + LastModifiedTimeMetadata.getLastModifiedTime(ImmutableList.of())); + } + + @Test + public void testGetLastModifiedTime_cacheEntry() throws IOException { + DescriptorDigest ignored = Mockito.mock(DescriptorDigest.class); + CacheEntry cacheEntry = + DefaultCacheEntry.builder() + .setLayerDigest(ignored) + .setLayerDiffId(ignored) + .setLayerSize(0) + .setLayerBlob(Blobs.from("ignored")) + .setMetadataBlob(Blobs.from(Instant.ofEpochMilli(1000).toString())) + .build(); + Assert.assertEquals( + FileTime.from(Instant.ofEpochMilli(1000)), + LastModifiedTimeMetadata.getLastModifiedTime(cacheEntry).orElseThrow(AssertionError::new)); + } + + @Test + public void testGetLastModifiedTime_cacheEntry_noMetadata() throws IOException { + DescriptorDigest ignored = Mockito.mock(DescriptorDigest.class); + CacheEntry cacheEntry = + DefaultCacheEntry.builder() + .setLayerDigest(ignored) + .setLayerDiffId(ignored) + .setLayerSize(0) + .setLayerBlob(Blobs.from("ignored")) + .build(); + Assert.assertFalse(LastModifiedTimeMetadata.getLastModifiedTime(cacheEntry).isPresent()); + } + + @Test + public void testGenerateMetadata() throws IOException { + Assert.assertEquals( + Instant.ofEpochMilli(2000).toString(), + Blobs.writeToString( + LastModifiedTimeMetadata.generateMetadata(ImmutableList.copyOf(layerEntries)))); + } +}