diff --git a/.idea/gradle.xml b/.idea/gradle.xml index 3d8c641e..b50ab4ad 100644 --- a/.idea/gradle.xml +++ b/.idea/gradle.xml @@ -29,6 +29,7 @@ diff --git a/app/build.gradle.kts b/app/build.gradle.kts index b5d06733..0b1ca05b 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -45,11 +45,14 @@ android { debug { isMinifyEnabled = false - + } } + kotlinOptions { + jvmTarget = "11" + } packagingOptions { - resources.excludes.addAll(listOf("META-INF/**","xsd/*","license/*")) + resources.excludes.addAll(listOf("META-INF/**", "xsd/*", "license/*")) resources.pickFirsts.add("kotlin/**") } compileOptions { @@ -72,6 +75,7 @@ dependencies { implementation(project(":core-api")) implementation(project(":functional")) + implementation(project(":snapshots")) implementation(project(":file-temp")) /* diff --git a/subprojects/snapshots/.gitignore b/subprojects/snapshots/.gitignore new file mode 100644 index 00000000..42afabfd --- /dev/null +++ b/subprojects/snapshots/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/subprojects/snapshots/build.gradle.kts b/subprojects/snapshots/build.gradle.kts new file mode 100644 index 00000000..376bcc67 --- /dev/null +++ b/subprojects/snapshots/build.gradle.kts @@ -0,0 +1,23 @@ +plugins { + id("java-library") +} + +java { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} + +dependencies { + api(project(":files")) + api(project(":hashing")) + + implementation(project(":base-annotations")) + + + implementation("com.google.guava:guava:30.1.1-jre") + implementation("com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava") + implementation("com.google.code.findbugs:jsr305:3.0.2") + + + implementation("org.slf4j:slf4j-api:1.7.36") +} \ No newline at end of file diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/RelativePathSupplier.java b/subprojects/snapshots/src/main/java/org/gradle/internal/RelativePathSupplier.java new file mode 100644 index 00000000..1ddb98af --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/RelativePathSupplier.java @@ -0,0 +1,27 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal; + +import java.util.Collection; + +public interface RelativePathSupplier { + boolean isRoot(); + + Collection getSegments(); + + String toRelativePath(); +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/CurrentFileCollectionFingerprint.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/CurrentFileCollectionFingerprint.java new file mode 100644 index 00000000..0a72e228 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/CurrentFileCollectionFingerprint.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint; + +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.snapshot.FileSystemSnapshot; + +/** + * A file collection fingerprint taken during this build. + */ +public interface CurrentFileCollectionFingerprint extends FileCollectionFingerprint { + /** + * Returns the combined hash of the contents of this {@link CurrentFileCollectionFingerprint}. + */ + HashCode getHash(); + + String getStrategyIdentifier(); + + /** + * Returns the snapshot used to capture these fingerprints. + */ + FileSystemSnapshot getSnapshot(); + + boolean isEmpty(); +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/DirectorySensitivity.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/DirectorySensitivity.java new file mode 100644 index 00000000..e5383add --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/DirectorySensitivity.java @@ -0,0 +1,47 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint; + +import org.gradle.internal.file.FileType; +import org.gradle.internal.snapshot.FileSystemLocationSnapshot; + +import java.util.function.Predicate; + +/** + * Specifies how a fingerprinter should handle directories that are found in a filecollection. + */ +public enum DirectorySensitivity { + /** + * Whatever the default behavior is for the given fingerprinter. For some fingerprinters, the + * default behavior is to fingerprint directories, for others, they ignore directories by default. + */ + DEFAULT(snapshot -> true), + /** + * Ignore directories + */ + IGNORE_DIRECTORIES(snapshot -> snapshot.getType() != FileType.Directory); + + private final Predicate fingerprintCheck; + + DirectorySensitivity(Predicate fingerprintCheck) { + this.fingerprintCheck = fingerprintCheck; + } + + public boolean shouldFingerprint(FileSystemLocationSnapshot snapshot) { + return fingerprintCheck.test(snapshot); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/FileCollectionFingerprint.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/FileCollectionFingerprint.java new file mode 100644 index 00000000..3fc4c666 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/FileCollectionFingerprint.java @@ -0,0 +1,79 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint; + +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedMap; +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.hash.Hashing; + +import java.util.Map; + +/** + * An immutable snapshot of some aspects of the contents and meta-data of a collection of files or directories. + */ +public interface FileCollectionFingerprint { + + /** + * The underlying fingerprints. + */ + Map getFingerprints(); + + /** + * The Merkle hashes of the roots which make up this file collection fingerprint. + */ + ImmutableMultimap getRootHashes(); + + /** + * The absolute paths for the roots of this file collection fingerprint. + */ + default ImmutableSet getRootPaths() { + return getRootHashes().keySet(); + } + + FileCollectionFingerprint EMPTY = new FileCollectionFingerprint() { + private final HashCode strategyConfigurationHash = Hashing.signature(getClass()); + + @Override + public Map getFingerprints() { + return ImmutableSortedMap.of(); + } + + @Override + public ImmutableMultimap getRootHashes() { + return ImmutableMultimap.of(); + } + + @Override + public ImmutableSet getRootPaths() { + return ImmutableSet.of(); + } + + @Override + public HashCode getStrategyConfigurationHash() { + return strategyConfigurationHash; + } + + @Override + public String toString() { + return "EMPTY"; + } + }; + + HashCode getStrategyConfigurationHash(); +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/FileSystemLocationFingerprint.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/FileSystemLocationFingerprint.java new file mode 100644 index 00000000..6439f776 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/FileSystemLocationFingerprint.java @@ -0,0 +1,39 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint; + +import org.gradle.internal.file.FileType; +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.hash.Hashable; +import org.gradle.internal.hash.Hashing; + +/** + * An immutable fingerprint of some aspects of a file's metadata and content. + * + * Should implement {@link #equals(Object)} and {@link #hashCode()} to compare these aspects. + * Comparisons are very frequent, so these methods need to be fast. + * + * File fingerprints are cached between builds, so their memory footprint should be kept to a minimum. + */ +public interface FileSystemLocationFingerprint extends Comparable, Hashable { + HashCode DIR_SIGNATURE = Hashing.signature("DIR"); + HashCode MISSING_FILE_SIGNATURE = Hashing.signature("MISSING"); + + String getNormalizedPath(); + HashCode getNormalizedContentHash(); + FileType getType(); +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/FingerprintHashingStrategy.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/FingerprintHashingStrategy.java new file mode 100644 index 00000000..03e3ec1b --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/FingerprintHashingStrategy.java @@ -0,0 +1,49 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint; + +import com.google.common.collect.ImmutableList; +import org.gradle.internal.hash.Hasher; + +import java.util.Collection; + +/** + * Strategy for appending a collection of fingerprints to a hasher. + */ +public enum FingerprintHashingStrategy { + SORT { + @Override + public void appendToHasher(Hasher hasher, Collection fingerprints) { + ImmutableList sortedFingerprints = ImmutableList.sortedCopyOf(fingerprints); + appendCollectionToHasherKeepingOrder(hasher, sortedFingerprints); + } + }, + KEEP_ORDER { + @Override + public void appendToHasher(Hasher hasher, Collection fingerprints) { + appendCollectionToHasherKeepingOrder(hasher, fingerprints); + } + }; + + public abstract void appendToHasher(Hasher hasher, Collection fingerprints); + + protected void appendCollectionToHasherKeepingOrder(Hasher hasher, Collection fingerprints) { + for (FileSystemLocationFingerprint fingerprint : fingerprints) { + fingerprint.appendToHasher(hasher); + } + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/FingerprintingStrategy.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/FingerprintingStrategy.java new file mode 100644 index 00000000..a4eb9f35 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/FingerprintingStrategy.java @@ -0,0 +1,55 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint; + +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.snapshot.FileSystemLocationSnapshot; +import org.gradle.internal.snapshot.FileSystemSnapshot; + +import java.util.Map; + +/** + * Strategy for converting a sequence of {@link FileSystemLocationSnapshot}s into a {@link FileCollectionFingerprint}. + */ +public interface FingerprintingStrategy { + + // TODO wolfs: Move these identifiers to the actual strategy classes when they live in :snapshots + String CLASSPATH_IDENTIFIER = "CLASSPATH"; + String COMPILE_CLASSPATH_IDENTIFIER = "COMPILE_CLASSPATH"; + + /** + * Converts the roots into the {@link FileSystemLocationFingerprint}s used by the {@link FileCollectionFingerprint}. + */ + Map collectFingerprints(FileSystemSnapshot roots); + + /** + * Used by the {@link FileCollectionFingerprint} to hash a map of fingerprints generated by {@link #collectFingerprints(FileSystemSnapshot)} + */ + FingerprintHashingStrategy getHashingStrategy(); + + /** + * UsedByScansPlugin + * Names are expected as part of org.gradle.api.internal.tasks.SnapshotTaskInputsBuildOperationType.Result.VisitState.getPropertyNormalizationStrategyName(). + */ + String getIdentifier(); + + CurrentFileCollectionFingerprint getEmptyFingerprint(); + + String normalizePath(FileSystemLocationSnapshot snapshot); + + HashCode getConfigurationHash(); +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/LineEndingSensitivity.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/LineEndingSensitivity.java new file mode 100644 index 00000000..557c223d --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/LineEndingSensitivity.java @@ -0,0 +1,22 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint; + +public enum LineEndingSensitivity { + DEFAULT, + NORMALIZE_LINE_ENDINGS; +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/ConfigurableNormalizer.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/ConfigurableNormalizer.java new file mode 100644 index 00000000..3c01ce95 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/ConfigurableNormalizer.java @@ -0,0 +1,28 @@ +/* + * Copyright 2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint.hashing; + +import org.gradle.internal.hash.Hasher; + +/** + * A resource normalizer which is configurable. + * + * Allows tracking changes to its configuration. + */ +public interface ConfigurableNormalizer { + void appendConfigurationToHasher(Hasher hasher); +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/FileSystemLocationSnapshotHasher.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/FileSystemLocationSnapshotHasher.java new file mode 100644 index 00000000..cb4c5381 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/FileSystemLocationSnapshotHasher.java @@ -0,0 +1,45 @@ +/* + * Copyright 2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint.hashing; + +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.hash.Hasher; +import org.gradle.internal.snapshot.FileSystemLocationSnapshot; + +import javax.annotation.Nullable; +import java.io.IOException; + +public interface FileSystemLocationSnapshotHasher extends ConfigurableNormalizer { + /** + * Returns {@code null} if the file should be ignored. + */ + @Nullable + HashCode hash(FileSystemLocationSnapshot snapshot) throws IOException; + + FileSystemLocationSnapshotHasher DEFAULT = new FileSystemLocationSnapshotHasher() { + @Nullable + @Override + public HashCode hash(FileSystemLocationSnapshot snapshot) { + return snapshot.getHash(); + } + + @Override + public void appendConfigurationToHasher(Hasher hasher) { + hasher.putString(getClass().getName()); + } + }; +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/RegularFileSnapshotContext.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/RegularFileSnapshotContext.java new file mode 100644 index 00000000..0336eb96 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/RegularFileSnapshotContext.java @@ -0,0 +1,27 @@ +/* + * Copyright 2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint.hashing; + +import org.gradle.internal.snapshot.RegularFileSnapshot; + +import java.util.function.Supplier; + +public interface RegularFileSnapshotContext { + Supplier getRelativePathSegments(); + + RegularFileSnapshot getSnapshot(); +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/RegularFileSnapshotContextHasher.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/RegularFileSnapshotContextHasher.java new file mode 100644 index 00000000..d4766967 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/RegularFileSnapshotContextHasher.java @@ -0,0 +1,31 @@ +/* + * Copyright 2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint.hashing; + +import org.gradle.internal.hash.HashCode; + +import javax.annotation.Nullable; +import java.io.IOException; + +public interface RegularFileSnapshotContextHasher { + + /** + * Returns {@code null} if the file should be ignored. + */ + @Nullable + HashCode hash(RegularFileSnapshotContext snapshotContext) throws IOException; +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/ResourceHasher.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/ResourceHasher.java new file mode 100644 index 00000000..c995a9c5 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/ResourceHasher.java @@ -0,0 +1,23 @@ +/* + * Copyright 2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint.hashing; + +/** + * Hashes resources (e.g., a class file in a jar or a class file in a directory) + */ +public interface ResourceHasher extends ConfigurableNormalizer, RegularFileSnapshotContextHasher, ZipEntryContextHasher { +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/ZipEntryContext.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/ZipEntryContext.java new file mode 100644 index 00000000..fb8d4c8c --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/ZipEntryContext.java @@ -0,0 +1,31 @@ +/* + * Copyright 2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint.hashing; + +import org.gradle.api.internal.file.archive.ZipEntry; + +import java.util.function.Supplier; + +public interface ZipEntryContext { + ZipEntry getEntry(); + + String getFullName(); + + String getRootParentName(); + + Supplier getRelativePathSegments(); +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/ZipEntryContextHasher.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/ZipEntryContextHasher.java new file mode 100644 index 00000000..16f8074c --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/ZipEntryContextHasher.java @@ -0,0 +1,33 @@ +/* + * Copyright 2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint.hashing; + +import org.gradle.internal.hash.HashCode; + +import javax.annotation.Nullable; +import java.io.IOException; + +/** + * Hashes a zip entry (e.g. a class file in a jar, a manifest file, a properties file) + */ +public interface ZipEntryContextHasher { + /** + * Returns {@code null} if the zip entry should be ignored. + */ + @Nullable + HashCode hash(ZipEntryContext zipEntryContext) throws IOException; +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/package-info.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/package-info.java new file mode 100644 index 00000000..69a144f1 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/hashing/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright 2021 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@NonNullApi +package org.gradle.internal.fingerprint.hashing; + +import org.gradle.api.NonNullApi; diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/AbsolutePathFingerprintingStrategy.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/AbsolutePathFingerprintingStrategy.java new file mode 100644 index 00000000..d2ecf2ea --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/AbsolutePathFingerprintingStrategy.java @@ -0,0 +1,83 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint.impl; + +import com.google.common.collect.ImmutableMap; +import org.gradle.internal.fingerprint.DirectorySensitivity; +import org.gradle.internal.fingerprint.FileSystemLocationFingerprint; +import org.gradle.internal.fingerprint.FingerprintHashingStrategy; +import org.gradle.internal.fingerprint.FingerprintingStrategy; +import org.gradle.internal.fingerprint.hashing.FileSystemLocationSnapshotHasher; +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.snapshot.FileSystemLocationSnapshot; +import org.gradle.internal.snapshot.FileSystemSnapshot; +import org.gradle.internal.snapshot.RootTrackingFileSystemSnapshotHierarchyVisitor; +import org.gradle.internal.snapshot.SnapshotVisitResult; + +import java.util.HashSet; +import java.util.Map; + +/** + * Fingerprint files without path or content normalization. + */ +public class AbsolutePathFingerprintingStrategy extends AbstractFingerprintingStrategy { + public static final FingerprintingStrategy DEFAULT = new AbsolutePathFingerprintingStrategy(DirectorySensitivity.DEFAULT); + public static final FingerprintingStrategy IGNORE_DIRECTORIES = new AbsolutePathFingerprintingStrategy(DirectorySensitivity.IGNORE_DIRECTORIES); + + public static final String IDENTIFIER = "ABSOLUTE_PATH"; + + private final FileSystemLocationSnapshotHasher normalizedContentHasher; + + public AbsolutePathFingerprintingStrategy(DirectorySensitivity directorySensitivity, FileSystemLocationSnapshotHasher normalizedContentHasher) { + super(IDENTIFIER, directorySensitivity, normalizedContentHasher); + this.normalizedContentHasher = normalizedContentHasher; + } + + private AbsolutePathFingerprintingStrategy(DirectorySensitivity directorySensitivity) { + this(directorySensitivity, FileSystemLocationSnapshotHasher.DEFAULT); + } + + @Override + public String normalizePath(FileSystemLocationSnapshot snapshot) { + return snapshot.getAbsolutePath(); + } + + @Override + public Map collectFingerprints(FileSystemSnapshot roots) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + HashSet processedEntries = new HashSet<>(); + roots.accept(new RootTrackingFileSystemSnapshotHierarchyVisitor() { + @Override + public SnapshotVisitResult visitEntry(FileSystemLocationSnapshot snapshot, boolean isRoot) { + String absolutePath = snapshot.getAbsolutePath(); + if (processedEntries.add(absolutePath) && getDirectorySensitivity().shouldFingerprint(snapshot)) { + HashCode normalizedContentHash = getNormalizedContentHash(snapshot, normalizedContentHasher); + if (normalizedContentHash != null) { + builder.put(absolutePath, new DefaultFileSystemLocationFingerprint(snapshot.getAbsolutePath(), snapshot.getType(), normalizedContentHash)); + } + } + return SnapshotVisitResult.CONTINUE; + } + }); + return builder.build(); + } + + @Override + public FingerprintHashingStrategy getHashingStrategy() { + return FingerprintHashingStrategy.SORT; + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/AbstractFingerprintingStrategy.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/AbstractFingerprintingStrategy.java new file mode 100644 index 00000000..6d448c30 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/AbstractFingerprintingStrategy.java @@ -0,0 +1,86 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint.impl; + +import org.gradle.internal.fingerprint.CurrentFileCollectionFingerprint; +import org.gradle.internal.fingerprint.DirectorySensitivity; +import org.gradle.internal.fingerprint.FingerprintingStrategy; +import org.gradle.internal.fingerprint.hashing.ConfigurableNormalizer; +import org.gradle.internal.fingerprint.hashing.FileSystemLocationSnapshotHasher; +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.hash.Hasher; +import org.gradle.internal.hash.Hashing; +import org.gradle.internal.snapshot.FileSystemLocationSnapshot; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.io.UncheckedIOException; + +public abstract class AbstractFingerprintingStrategy implements FingerprintingStrategy { + private final String identifier; + private final CurrentFileCollectionFingerprint emptyFingerprint; + private final DirectorySensitivity directorySensitivity; + private final HashCode configurationHash; + + public AbstractFingerprintingStrategy( + String identifier, + DirectorySensitivity directorySensitivity, + ConfigurableNormalizer contentNormalizer + ) { + this.identifier = identifier; + this.emptyFingerprint = new EmptyCurrentFileCollectionFingerprint(identifier); + this.directorySensitivity = directorySensitivity; + Hasher hasher = Hashing.newHasher(); + hasher.putString(getClass().getName()); + contentNormalizer.appendConfigurationToHasher(hasher); + this.configurationHash = hasher.hash(); + } + + @Override + public String getIdentifier() { + return identifier; + } + + @Override + public CurrentFileCollectionFingerprint getEmptyFingerprint() { + return emptyFingerprint; + } + + @Nullable + protected HashCode getNormalizedContentHash(FileSystemLocationSnapshot snapshot, FileSystemLocationSnapshotHasher normalizedContentHasher) { + try { + return normalizedContentHasher.hash(snapshot); + } catch (IOException e) { + throw new UncheckedIOException(failedToNormalize(snapshot), e); + } catch (UncheckedIOException e) { + throw new UncheckedIOException(failedToNormalize(snapshot), e.getCause()); + } + } + + private static String failedToNormalize(FileSystemLocationSnapshot snapshot) { + return String.format("Failed to normalize content of '%s'.", snapshot.getAbsolutePath()); + } + + public DirectorySensitivity getDirectorySensitivity() { + return directorySensitivity; + } + + @Override + public HashCode getConfigurationHash() { + return configurationHash; + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/DefaultCurrentFileCollectionFingerprint.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/DefaultCurrentFileCollectionFingerprint.java new file mode 100644 index 00000000..07ce00fd --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/DefaultCurrentFileCollectionFingerprint.java @@ -0,0 +1,130 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint.impl; + +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.Iterables; +import org.gradle.internal.fingerprint.CurrentFileCollectionFingerprint; +import org.gradle.internal.fingerprint.FileCollectionFingerprint; +import org.gradle.internal.fingerprint.FileSystemLocationFingerprint; +import org.gradle.internal.fingerprint.FingerprintHashingStrategy; +import org.gradle.internal.fingerprint.FingerprintingStrategy; +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.hash.Hasher; +import org.gradle.internal.hash.Hashing; +import org.gradle.internal.snapshot.FileSystemSnapshot; +import org.gradle.internal.snapshot.SnapshotUtil; + +import javax.annotation.Nullable; +import java.util.Map; + +public class DefaultCurrentFileCollectionFingerprint implements CurrentFileCollectionFingerprint { + + private final Map fingerprints; + private final FingerprintHashingStrategy hashingStrategy; + private final String identifier; + private final FileSystemSnapshot roots; + private final ImmutableMultimap rootHashes; + private final HashCode strategyConfigurationHash; + private HashCode hash; + + public static CurrentFileCollectionFingerprint from(FileSystemSnapshot roots, FingerprintingStrategy strategy, @Nullable FileCollectionFingerprint candidate) { + if (roots == FileSystemSnapshot.EMPTY) { + return strategy.getEmptyFingerprint(); + } + + ImmutableMultimap rootHashes = SnapshotUtil.getRootHashes(roots); + Map fingerprints; + if (candidate != null + && candidate.getStrategyConfigurationHash().equals(strategy.getConfigurationHash()) + && equalRootHashes(candidate.getRootHashes(), rootHashes) + ) { + fingerprints = candidate.getFingerprints(); + } else { + fingerprints = strategy.collectFingerprints(roots); + } + if (fingerprints.isEmpty()) { + return strategy.getEmptyFingerprint(); + } + return new DefaultCurrentFileCollectionFingerprint(fingerprints, roots, rootHashes, strategy); + } + + private static boolean equalRootHashes(ImmutableMultimap first, ImmutableMultimap second) { + // We cannot use `first.equals(second)`, since the order of the root hashes matters + return Iterables.elementsEqual(first.entries(), second.entries()); + } + + private DefaultCurrentFileCollectionFingerprint( + Map fingerprints, + FileSystemSnapshot roots, + ImmutableMultimap rootHashes, + FingerprintingStrategy strategy + ) { + this.fingerprints = fingerprints; + this.identifier = strategy.getIdentifier(); + this.hashingStrategy = strategy.getHashingStrategy(); + this.strategyConfigurationHash = strategy.getConfigurationHash(); + this.roots = roots; + this.rootHashes = rootHashes; + } + + @Override + public HashCode getHash() { + if (hash == null) { + Hasher hasher = Hashing.newHasher(); + hashingStrategy.appendToHasher(hasher, fingerprints.values()); + hash = hasher.hash(); + } + return hash; + } + + @Override + public boolean isEmpty() { + // We'd have created an EmptyCurrentFileCollectionFingerprint if there were no file fingerprints + return false; + } + + @Override + public Map getFingerprints() { + return fingerprints; + } + + @Override + public ImmutableMultimap getRootHashes() { + return rootHashes; + } + + @Override + public String getStrategyIdentifier() { + return identifier; + } + + @Override + public FileSystemSnapshot getSnapshot() { + return roots; + } + + @Override + public HashCode getStrategyConfigurationHash() { + return strategyConfigurationHash; + } + + @Override + public String toString() { + return identifier + fingerprints; + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/DefaultFileSystemLocationFingerprint.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/DefaultFileSystemLocationFingerprint.java new file mode 100644 index 00000000..27934c2f --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/DefaultFileSystemLocationFingerprint.java @@ -0,0 +1,129 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint.impl; + +import org.gradle.internal.file.FileType; +import org.gradle.internal.fingerprint.FileSystemLocationFingerprint; +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.hash.Hasher; +import org.gradle.internal.snapshot.FileSystemLocationSnapshot; + +public class DefaultFileSystemLocationFingerprint implements FileSystemLocationFingerprint { + private final HashCode normalizedContentHash; + private final String normalizedPath; + + public DefaultFileSystemLocationFingerprint(String normalizedPath, FileType type, HashCode contentHash) { + this.normalizedContentHash = hashForType(type, contentHash); + this.normalizedPath = normalizedPath; + } + + public DefaultFileSystemLocationFingerprint(String normalizedPath, FileSystemLocationSnapshot snapshot) { + this(normalizedPath, snapshot.getType(), snapshot.getHash()); + } + + private static HashCode hashForType(FileType fileType, HashCode hash) { + switch (fileType) { + case Directory: + return DIR_SIGNATURE; + case Missing: + return MISSING_FILE_SIGNATURE; + case RegularFile: + return hash; + default: + throw new IllegalStateException("Unknown file type: " + fileType); + } + } + + @Override + public final void appendToHasher(Hasher hasher) { + hasher.putString(getNormalizedPath()); + hasher.putHash(getNormalizedContentHash()); + } + + @Override + public FileType getType() { + if (normalizedContentHash == DIR_SIGNATURE) { + return FileType.Directory; + } else if (normalizedContentHash == MISSING_FILE_SIGNATURE) { + return FileType.Missing; + } else { + return FileType.RegularFile; + } + } + + @Override + public String getNormalizedPath() { + return normalizedPath; + } + + @Override + public HashCode getNormalizedContentHash() { + return normalizedContentHash; + } + + @Override + public final int compareTo(FileSystemLocationFingerprint o) { + int result = getNormalizedPath().compareTo(o.getNormalizedPath()); + if (result == 0) { + result = getNormalizedContentHash().compareTo(o.getNormalizedContentHash()); + } + return result; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + DefaultFileSystemLocationFingerprint that = (DefaultFileSystemLocationFingerprint) o; + + if (!normalizedContentHash.equals(that.normalizedContentHash)) { + return false; + } + return normalizedPath.equals(that.normalizedPath); + } + + @Override + public int hashCode() { + int result = normalizedContentHash.hashCode(); + result = 31 * result + normalizedPath.hashCode(); + return result; + } + + @Override + public String toString() { + return String.format("'%s' / %s", + getNormalizedPath(), + getHashOrTypeToDisplay() + ); + } + + private Object getHashOrTypeToDisplay() { + switch (getType()) { + case Directory: + return "DIR"; + case Missing: + return "MISSING"; + default: + return normalizedContentHash; + } + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/EmptyCurrentFileCollectionFingerprint.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/EmptyCurrentFileCollectionFingerprint.java new file mode 100644 index 00000000..fc31b0eb --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/EmptyCurrentFileCollectionFingerprint.java @@ -0,0 +1,84 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint.impl; + +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import org.gradle.internal.fingerprint.CurrentFileCollectionFingerprint; +import org.gradle.internal.fingerprint.FileSystemLocationFingerprint; +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.hash.Hashing; +import org.gradle.internal.snapshot.FileSystemSnapshot; + +import java.util.Collections; +import java.util.Map; + +public class EmptyCurrentFileCollectionFingerprint implements CurrentFileCollectionFingerprint { + + private static final HashCode SIGNATURE = Hashing.signature(EmptyCurrentFileCollectionFingerprint.class); + + private final String identifier; + + public EmptyCurrentFileCollectionFingerprint(String identifier) { + this.identifier = identifier; + } + + @Override + public HashCode getHash() { + return SIGNATURE; + } + + @Override + public boolean isEmpty() { + return true; + } + + @Override + public Map getFingerprints() { + return Collections.emptyMap(); + } + + @Override + public FileSystemSnapshot getSnapshot() { + return FileSystemSnapshot.EMPTY; + } + + @Override + public ImmutableMultimap getRootHashes() { + return ImmutableMultimap.of(); + } + + @Override + public ImmutableSet getRootPaths() { + return ImmutableSet.of(); + } + + @Override + public HashCode getStrategyConfigurationHash() { + return SIGNATURE; + } + + @Override + public String getStrategyIdentifier() { + return identifier; + } + + @Override + public String toString() { + return identifier + "{EMPTY}"; + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/IgnoredPathFileSystemLocationFingerprint.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/IgnoredPathFileSystemLocationFingerprint.java new file mode 100644 index 00000000..2e9f35ab --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/IgnoredPathFileSystemLocationFingerprint.java @@ -0,0 +1,99 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint.impl; + +import org.gradle.internal.file.FileType; +import org.gradle.internal.fingerprint.FileSystemLocationFingerprint; +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.hash.Hasher; + +public class IgnoredPathFileSystemLocationFingerprint implements FileSystemLocationFingerprint { + + public static final IgnoredPathFileSystemLocationFingerprint DIRECTORY = new IgnoredPathFileSystemLocationFingerprint(FileType.Directory, DIR_SIGNATURE); + private static final IgnoredPathFileSystemLocationFingerprint MISSING_FILE = new IgnoredPathFileSystemLocationFingerprint(FileType.Missing, MISSING_FILE_SIGNATURE); + + private final FileType type; + private final HashCode normalizedContentHash; + + public static IgnoredPathFileSystemLocationFingerprint create(FileType type, HashCode contentHash) { + switch (type) { + case Directory: + return DIRECTORY; + case Missing: + return MISSING_FILE; + case RegularFile: + return new IgnoredPathFileSystemLocationFingerprint(FileType.RegularFile, contentHash); + default: + throw new IllegalStateException(); + } + } + + private IgnoredPathFileSystemLocationFingerprint(FileType type, HashCode normalizedContentHash) { + this.type = type; + this.normalizedContentHash = normalizedContentHash; + } + + @Override + public String getNormalizedPath() { + return ""; + } + + @Override + public HashCode getNormalizedContentHash() { + return normalizedContentHash; + } + + @Override + public FileType getType() { + return type; + } + + @Override + public int compareTo(FileSystemLocationFingerprint o) { + if (!(o instanceof IgnoredPathFileSystemLocationFingerprint)) { + return -1; + } + return normalizedContentHash.compareTo(o.getNormalizedContentHash()); + } + + @Override + public void appendToHasher(Hasher hasher) { + hasher.putHash(normalizedContentHash); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + IgnoredPathFileSystemLocationFingerprint that = (IgnoredPathFileSystemLocationFingerprint) o; + return normalizedContentHash.equals(that.normalizedContentHash); + } + + @Override + public int hashCode() { + return normalizedContentHash.hashCode(); + } + + @Override + public String toString() { + return String.format("IGNORED / %s", getType() == FileType.Directory ? "DIR" : getType() == FileType.Missing ? "MISSING" : normalizedContentHash); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/IgnoredPathFingerprintingStrategy.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/IgnoredPathFingerprintingStrategy.java new file mode 100644 index 00000000..b01feac1 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/IgnoredPathFingerprintingStrategy.java @@ -0,0 +1,96 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint.impl; + +import com.google.common.collect.ImmutableMap; +import org.gradle.internal.fingerprint.DirectorySensitivity; +import org.gradle.internal.fingerprint.FileSystemLocationFingerprint; +import org.gradle.internal.fingerprint.FingerprintHashingStrategy; +import org.gradle.internal.fingerprint.hashing.FileSystemLocationSnapshotHasher; +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.snapshot.FileSystemLocationSnapshot; +import org.gradle.internal.snapshot.FileSystemLocationSnapshot.FileSystemLocationSnapshotVisitor; +import org.gradle.internal.snapshot.FileSystemSnapshot; +import org.gradle.internal.snapshot.MissingFileSnapshot; +import org.gradle.internal.snapshot.RegularFileSnapshot; +import org.gradle.internal.snapshot.SnapshotVisitResult; + +import java.util.HashSet; +import java.util.Map; + +/** + * Fingerprint files ignoring the path. + * + * Ignores directories. + */ +public class IgnoredPathFingerprintingStrategy extends AbstractFingerprintingStrategy { + public static final IgnoredPathFingerprintingStrategy DEFAULT = new IgnoredPathFingerprintingStrategy(); + public static final String IDENTIFIER = "IGNORED_PATH"; + public static final String IGNORED_PATH = ""; + + private final FileSystemLocationSnapshotHasher normalizedContentHasher; + + public IgnoredPathFingerprintingStrategy(FileSystemLocationSnapshotHasher normalizedContentHasher) { + super(IDENTIFIER, DirectorySensitivity.DEFAULT, normalizedContentHasher); + this.normalizedContentHasher = normalizedContentHasher; + } + + private IgnoredPathFingerprintingStrategy() { + this(FileSystemLocationSnapshotHasher.DEFAULT); + } + + @Override + public String normalizePath(FileSystemLocationSnapshot snapshot) { + return IGNORED_PATH; + } + + @Override + public Map collectFingerprints(FileSystemSnapshot roots) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + HashSet processedEntries = new HashSet<>(); + roots.accept(snapshot -> { + snapshot.accept(new FileSystemLocationSnapshotVisitor() { + @Override + public void visitRegularFile(RegularFileSnapshot fileSnapshot) { + visitNonDirectoryEntry(snapshot); + } + + @Override + public void visitMissing(MissingFileSnapshot missingSnapshot) { + visitNonDirectoryEntry(snapshot); + } + + private void visitNonDirectoryEntry(FileSystemLocationSnapshot snapshot) { + String absolutePath = snapshot.getAbsolutePath(); + if (processedEntries.add(absolutePath)) { + HashCode normalizedContentHash = getNormalizedContentHash(snapshot, normalizedContentHasher); + if (normalizedContentHash != null) { + builder.put(absolutePath, IgnoredPathFileSystemLocationFingerprint.create(snapshot.getType(), normalizedContentHash)); + } + } + } + }); + return SnapshotVisitResult.CONTINUE; + }); + return builder.build(); + } + + @Override + public FingerprintHashingStrategy getHashingStrategy() { + return FingerprintHashingStrategy.SORT; + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/NameOnlyFingerprintingStrategy.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/NameOnlyFingerprintingStrategy.java new file mode 100644 index 00000000..df8c49f6 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/NameOnlyFingerprintingStrategy.java @@ -0,0 +1,87 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint.impl; + +import com.google.common.collect.ImmutableMap; +import org.gradle.internal.file.FileType; +import org.gradle.internal.fingerprint.DirectorySensitivity; +import org.gradle.internal.fingerprint.FileSystemLocationFingerprint; +import org.gradle.internal.fingerprint.FingerprintHashingStrategy; +import org.gradle.internal.fingerprint.hashing.FileSystemLocationSnapshotHasher; +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.snapshot.FileSystemLocationSnapshot; +import org.gradle.internal.snapshot.FileSystemSnapshot; +import org.gradle.internal.snapshot.RootTrackingFileSystemSnapshotHierarchyVisitor; +import org.gradle.internal.snapshot.SnapshotVisitResult; + +import java.util.HashSet; +import java.util.Map; + +/** + * Fingerprint files normalizing the path to the file name. + * + * File names for root directories are ignored. + */ +public class NameOnlyFingerprintingStrategy extends AbstractFingerprintingStrategy { + public static final NameOnlyFingerprintingStrategy DEFAULT = new NameOnlyFingerprintingStrategy(DirectorySensitivity.DEFAULT); + public static final NameOnlyFingerprintingStrategy IGNORE_DIRECTORIES = new NameOnlyFingerprintingStrategy(DirectorySensitivity.IGNORE_DIRECTORIES); + public static final String IDENTIFIER = "NAME_ONLY"; + private final FileSystemLocationSnapshotHasher normalizedContentHasher; + + public NameOnlyFingerprintingStrategy(DirectorySensitivity directorySensitivity, FileSystemLocationSnapshotHasher normalizedContentHasher) { + super(IDENTIFIER, directorySensitivity, normalizedContentHasher); + this.normalizedContentHasher = normalizedContentHasher; + } + + private NameOnlyFingerprintingStrategy(DirectorySensitivity directorySensitivity) { + this(directorySensitivity, FileSystemLocationSnapshotHasher.DEFAULT); + } + + @Override + public String normalizePath(FileSystemLocationSnapshot snapshot) { + return snapshot.getName(); + } + + @Override + public Map collectFingerprints(FileSystemSnapshot roots) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + HashSet processedEntries = new HashSet<>(); + roots.accept(new RootTrackingFileSystemSnapshotHierarchyVisitor() { + @Override + public SnapshotVisitResult visitEntry(FileSystemLocationSnapshot snapshot, boolean isRoot) { + String absolutePath = snapshot.getAbsolutePath(); + if (processedEntries.add(absolutePath) && getDirectorySensitivity().shouldFingerprint(snapshot)) { + if (isRoot && snapshot.getType() == FileType.Directory) { + builder.put(absolutePath, IgnoredPathFileSystemLocationFingerprint.DIRECTORY); + } else { + HashCode normalizedContentHash = getNormalizedContentHash(snapshot, normalizedContentHasher); + if (normalizedContentHash != null) { + builder.put(absolutePath, new DefaultFileSystemLocationFingerprint(snapshot.getName(), snapshot.getType(), normalizedContentHash)); + } + } + } + return SnapshotVisitResult.CONTINUE; + } + }); + return builder.build(); + } + + @Override + public FingerprintHashingStrategy getHashingStrategy() { + return FingerprintHashingStrategy.SORT; + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/RelativePathFingerprintingStrategy.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/RelativePathFingerprintingStrategy.java new file mode 100644 index 00000000..68d33a7f --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/RelativePathFingerprintingStrategy.java @@ -0,0 +1,103 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.fingerprint.impl; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Interner; +import org.gradle.internal.file.FileType; +import org.gradle.internal.fingerprint.DirectorySensitivity; +import org.gradle.internal.fingerprint.FileSystemLocationFingerprint; +import org.gradle.internal.fingerprint.FingerprintHashingStrategy; +import org.gradle.internal.fingerprint.hashing.FileSystemLocationSnapshotHasher; +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.snapshot.FileSystemLocationSnapshot; +import org.gradle.internal.snapshot.FileSystemSnapshot; +import org.gradle.internal.snapshot.RelativePathTracker; +import org.gradle.internal.snapshot.SnapshotVisitResult; + +import javax.annotation.Nullable; +import java.util.HashSet; +import java.util.Map; + +/** + * Fingerprint file system snapshots normalizing the path to the relative path in a hierarchy. + * + * File names for root directories are ignored. For root files, the file name is used as normalized path. + */ +public class RelativePathFingerprintingStrategy extends AbstractFingerprintingStrategy { + public static final String IDENTIFIER = "RELATIVE_PATH"; + + private final Interner stringInterner; + private final FileSystemLocationSnapshotHasher normalizedContentHasher; + + public RelativePathFingerprintingStrategy(Interner stringInterner, DirectorySensitivity directorySensitivity, FileSystemLocationSnapshotHasher normalizedContentHasher) { + super(IDENTIFIER, directorySensitivity, normalizedContentHasher); + this.stringInterner = stringInterner; + this.normalizedContentHasher = normalizedContentHasher; + } + + public RelativePathFingerprintingStrategy(Interner stringInterner, DirectorySensitivity directorySensitivity) { + this(stringInterner, directorySensitivity, FileSystemLocationSnapshotHasher.DEFAULT); + } + + @Override + public String normalizePath(FileSystemLocationSnapshot snapshot) { + if (snapshot.getType() == FileType.Directory) { + return ""; + } else { + return snapshot.getName(); + } + } + + @Override + public Map collectFingerprints(FileSystemSnapshot roots) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + HashSet processedEntries = new HashSet<>(); + roots.accept(new RelativePathTracker(), (snapshot, relativePath) -> { + String absolutePath = snapshot.getAbsolutePath(); + if (processedEntries.add(absolutePath) && getDirectorySensitivity().shouldFingerprint(snapshot)) { + FileSystemLocationFingerprint fingerprint; + if (relativePath.isRoot()) { + if (snapshot.getType() == FileType.Directory) { + fingerprint = IgnoredPathFileSystemLocationFingerprint.DIRECTORY; + } else { + fingerprint = fingerprint(snapshot.getName(), snapshot.getType(), snapshot); + } + } else { + fingerprint = fingerprint(stringInterner.intern(relativePath.toRelativePath()), snapshot.getType(), snapshot); + } + + if (fingerprint != null) { + builder.put(absolutePath, fingerprint); + } + } + return SnapshotVisitResult.CONTINUE; + }); + return builder.build(); + } + + @Nullable + FileSystemLocationFingerprint fingerprint(String name, FileType type, FileSystemLocationSnapshot snapshot) { + HashCode normalizedContentHash = getNormalizedContentHash(snapshot, normalizedContentHasher); + return normalizedContentHash == null ? null : new DefaultFileSystemLocationFingerprint(name, type, normalizedContentHash); + } + + @Override + public FingerprintHashingStrategy getHashingStrategy() { + return FingerprintHashingStrategy.SORT; + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/package-info.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/package-info.java new file mode 100644 index 00000000..a560448f --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/impl/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@NonNullApi +package org.gradle.internal.fingerprint.impl; + +import org.gradle.api.NonNullApi; diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/package-info.java b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/package-info.java new file mode 100644 index 00000000..ea215d8d --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/fingerprint/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@NonNullApi +package org.gradle.internal.fingerprint; + +import org.gradle.api.NonNullApi; diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/isolation/Isolatable.java b/subprojects/snapshots/src/main/java/org/gradle/internal/isolation/Isolatable.java new file mode 100644 index 00000000..37b7cdd2 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/isolation/Isolatable.java @@ -0,0 +1,52 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.isolation; + +import org.gradle.internal.hash.Hashable; +import org.gradle.internal.snapshot.ValueSnapshot; + +import javax.annotation.Nullable; + +/** + * Isolatable objects can return an isolated instance of the given type T from which this object was created. + * An isolated instance has the same internal state as the original object on which this isolatable was based, + * but it is guaranteed not to retain any references to mutable state from the original instance. + *

+ * The primary reason to need such an isolated instance of an object is to ensure that work can be done in parallel using the instance without + * fear that its internal state is changing while the work is being carried out. + */ +public interface Isolatable extends Hashable { + /** + * Returns this value as a {@link ValueSnapshot}. The returned value should not hold any references to user ClassLoaders. + */ + ValueSnapshot asSnapshot(); + + /** + * Returns an instance of T that is isolated from the original object and all other instances. + * When T is mutable, a new instance is created on each call. When T is immutable, a new instance may or may not be created on each call. This may potentially be expensive. + */ + @Nullable + T isolate(); + + /** + * Returns an instance of S constructed from the state of the original object, if possible. + * + * @return null if not supported, or the value is null. + */ + @Nullable + S coerce(Class type); +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/isolation/IsolatableFactory.java b/subprojects/snapshots/src/main/java/org/gradle/internal/isolation/IsolatableFactory.java new file mode 100644 index 00000000..b9a39aeb --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/isolation/IsolatableFactory.java @@ -0,0 +1,26 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.isolation; + +import javax.annotation.Nullable; + +public interface IsolatableFactory { + /** + * Creates an {@link Isolatable} that reflects the current state of the given value. Any changes made to the value will not be visible to the {@link Isolatable} and vice versa. + */ + Isolatable isolate(@Nullable T value); +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/isolation/package-info.java b/subprojects/snapshots/src/main/java/org/gradle/internal/isolation/package-info.java new file mode 100644 index 00000000..9831e186 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/isolation/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@NonNullApi +package org.gradle.internal.isolation; + +import org.gradle.api.NonNullApi; diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/AbstractFileSystemLocationSnapshot.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/AbstractFileSystemLocationSnapshot.java new file mode 100644 index 00000000..9b826162 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/AbstractFileSystemLocationSnapshot.java @@ -0,0 +1,136 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import org.gradle.internal.file.FileMetadata.AccessType; + +import java.util.Optional; + +public abstract class AbstractFileSystemLocationSnapshot implements FileSystemLocationSnapshot { + private final String absolutePath; + private final String name; + private final AccessType accessType; + + public AbstractFileSystemLocationSnapshot(String absolutePath, String name, AccessType accessType) { + this.absolutePath = absolutePath; + this.name = name; + this.accessType = accessType; + } + + protected static MissingFileSnapshot missingSnapshotForAbsolutePath(String filePath) { + return new MissingFileSnapshot(filePath, AccessType.DIRECT); + } + + @Override + public String getAbsolutePath() { + return absolutePath; + } + + @Override + public String getName() { + return name; + } + + @Override + public AccessType getAccessType() { + return accessType; + } + + public String getPathToParent() { + return getName(); + } + + @Override + public FileSystemLocationSnapshot store(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, MetadataSnapshot snapshot, SnapshotHierarchy.NodeDiffListener diffListener) { + return this; + } + + @Override + public void accept(SnapshotHierarchy.SnapshotVisitor snapshotVisitor) { + snapshotVisitor.visitSnapshotRoot(this); + } + + @Override + public boolean hasDescendants() { + return true; + } + + @Override + public FileSystemNode asFileSystemNode() { + return this; + } + + @Override + public Optional getSnapshot() { + return Optional.of(this); + } + + @Override + public Optional getSnapshot(VfsRelativePath relativePath, CaseSensitivity caseSensitivity) { + return getChildSnapshot(relativePath, caseSensitivity); + } + + protected Optional getChildSnapshot(VfsRelativePath relativePath, CaseSensitivity caseSensitivity) { + return Optional.of(missingSnapshotForAbsolutePath(relativePath.getAbsolutePath())); + } + + @Override + public ReadOnlyFileSystemNode getNode(VfsRelativePath relativePath, CaseSensitivity caseSensitivity) { + return getChildNode(relativePath, caseSensitivity); + } + + protected ReadOnlyFileSystemNode getChildNode(VfsRelativePath relativePath, CaseSensitivity caseSensitivity) { + return missingSnapshotForAbsolutePath(relativePath.getAbsolutePath()); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + AbstractFileSystemLocationSnapshot that = (AbstractFileSystemLocationSnapshot) o; + + if (accessType != that.accessType) { + return false; + } + if (!name.equals(that.name)) { + return false; + } + if (!absolutePath.equals(that.absolutePath)) { + return false; + } + return getHash().equals(that.getHash()); + } + + @Override + public int hashCode() { + int result = absolutePath.hashCode(); + result = 31 * result + name.hashCode(); + result = 31 * result + accessType.hashCode(); + result = 31 * result + getHash().hashCode(); + return result; + } + + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/AbstractIncompleteFileSystemNode.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/AbstractIncompleteFileSystemNode.java new file mode 100644 index 00000000..63e943bf --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/AbstractIncompleteFileSystemNode.java @@ -0,0 +1,159 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import org.gradle.internal.file.FileType; + +import java.util.Optional; +import java.util.function.Predicate; + +public abstract class AbstractIncompleteFileSystemNode implements FileSystemNode { + protected final ChildMap children; + + @SuppressWarnings("unchecked") + public AbstractIncompleteFileSystemNode(ChildMap children) { + this.children = (ChildMap) children; + } + + @Override + public ReadOnlyFileSystemNode getNode(VfsRelativePath targetPath, CaseSensitivity caseSensitivity) { + return SnapshotUtil.getChild(children, targetPath, caseSensitivity); + } + + @Override + public Optional invalidate(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, SnapshotHierarchy.NodeDiffListener diffListener) { + ChildMap newChildren = children.invalidate(targetPath, caseSensitivity, new ChildMap.InvalidationHandler() { + @Override + public Optional handleAsDescendantOfChild(VfsRelativePath pathInChild, FileSystemNode child) { + return child.invalidate(pathInChild, caseSensitivity, diffListener); + } + + @Override + public void handleAsAncestorOfChild(String childPath, FileSystemNode child) { + diffListener.nodeRemoved(child); + } + + @Override + public void handleExactMatchWithChild(FileSystemNode child) { + diffListener.nodeRemoved(child); + } + + @Override + public void handleUnrelatedToAnyChild() { + } + }); + if (newChildren.isEmpty()) { + return withAllChildrenRemoved(); + } + if (newChildren == children) { + return Optional.of(withIncompleteChildren()); + } + return Optional.of(withIncompleteChildren(newChildren)); + } + + @Override + public FileSystemNode store(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, MetadataSnapshot snapshot, SnapshotHierarchy.NodeDiffListener diffListener) { + ChildMap newChildren = children.store(targetPath, caseSensitivity, new ChildMap.StoreHandler() { + @Override + public FileSystemNode handleAsDescendantOfChild(VfsRelativePath pathInChild, FileSystemNode child) { + return child.store(pathInChild, caseSensitivity, snapshot, diffListener); + } + + @Override + public FileSystemNode handleAsAncestorOfChild(String childPath, FileSystemNode child) { + FileSystemNode newChild = snapshot.asFileSystemNode(); + diffListener.nodeRemoved(child); + diffListener.nodeAdded(newChild); + return newChild; + } + + @Override + public FileSystemNode mergeWithExisting(FileSystemNode child) { + if (snapshot instanceof FileSystemLocationSnapshot || !child.getSnapshot().map(oldSnapshot -> oldSnapshot instanceof FileSystemLocationSnapshot).orElse(false)) { + FileSystemNode newChild = snapshot.asFileSystemNode(); + diffListener.nodeRemoved(child); + diffListener.nodeAdded(newChild); + return newChild; + } else { + return child; + } + } + + @Override + public FileSystemNode createChild() { + FileSystemNode newChild = snapshot.asFileSystemNode(); + diffListener.nodeAdded(newChild); + return newChild; + } + + @Override + public FileSystemNode createNodeFromChildren(ChildMap children) { + boolean isDirectory = anyChildMatches(children, node -> node.getSnapshot().map(this::isRegularFileOrDirectory).orElse(false)); + return isDirectory ? new PartialDirectoryNode(children) : new UnknownFileSystemNode(children); + } + + private boolean isRegularFileOrDirectory(MetadataSnapshot metadataSnapshot) { + return metadataSnapshot.getType() != FileType.Missing; + } + }); + if (newChildren == children) { + return this; + } + return withIncompleteChildren(newChildren); + } + + @Override + public Optional getSnapshot(VfsRelativePath targetPath, CaseSensitivity caseSensitivity) { + return SnapshotUtil.getMetadataFromChildren(children, targetPath, caseSensitivity, Optional::empty); + } + + /** + * Returns an updated node with the same children. The list of children are + * incomplete, even if they were complete before. + */ + protected abstract FileSystemNode withIncompleteChildren(); + + /** + * Returns an updated node with an updated list of children. + * + * Caller must ensure the child list is not be mutated as the method + * doesn't make a defensive copy. + */ + protected abstract FileSystemNode withIncompleteChildren(ChildMap newChildren); + + /** + * Returns an updated node with all children removed, or {@link Optional#empty()} + * if the node without children would contain no useful information to keep around. + */ + protected abstract Optional withAllChildrenRemoved(); + + @Override + public void accept(SnapshotHierarchy.SnapshotVisitor snapshotVisitor) { + children.visitChildren((childPath, child) -> child.accept(snapshotVisitor)); + } + + @Override + public boolean hasDescendants() { + return anyChildMatches(children, FileSystemNode::hasDescendants); + } + + private static boolean anyChildMatches(ChildMap children, Predicate predicate) { + return children.entries().stream() + .map(ChildMap.Entry::getValue) + .anyMatch(predicate); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/AbstractInvalidateChildHandler.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/AbstractInvalidateChildHandler.java new file mode 100644 index 00000000..c0ab8402 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/AbstractInvalidateChildHandler.java @@ -0,0 +1,62 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import java.util.Optional; + +public abstract class AbstractInvalidateChildHandler implements ChildMap.NodeHandler> { + + private final ChildMap.InvalidationHandler handler; + + public AbstractInvalidateChildHandler(ChildMap.InvalidationHandler handler) { + this.handler = handler; + } + + public abstract ChildMap getChildMap(); + + public abstract ChildMap withReplacedChild(RESULT newChild); + + public abstract ChildMap withReplacedChild(String newChildPath, RESULT newChild); + + public abstract ChildMap withRemovedChild(); + + @Override + public ChildMap handleAsDescendantOfChild(VfsRelativePath pathInChild, T child) { + Optional invalidatedChild = handler.handleAsDescendantOfChild(pathInChild, child); + return invalidatedChild + .map(this::withReplacedChild) + .orElseGet(this::withRemovedChild); + } + + @Override + public ChildMap handleAsAncestorOfChild(String childPath, T child) { + handler.handleAsAncestorOfChild(childPath, child); + return withRemovedChild(); + } + + @Override + public ChildMap handleExactMatchWithChild(T child) { + handler.handleExactMatchWithChild(child); + return withRemovedChild(); + } + + @Override + public ChildMap handleUnrelatedToAnyChild() { + handler.handleUnrelatedToAnyChild(); + return getChildMap(); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/AbstractListChildMap.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/AbstractListChildMap.java new file mode 100644 index 00000000..a4d4bed9 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/AbstractListChildMap.java @@ -0,0 +1,169 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +public abstract class AbstractListChildMap implements ChildMap { + protected final List> entries; + + protected AbstractListChildMap(List> entries) { + this.entries = entries; + } + + @Override + public boolean isEmpty() { + return false; + } + + @Override + public List values() { + return entries.stream() + .map(Entry::getValue) + .collect(Collectors.toList()); + } + + @Override + public List> entries() { + return entries; + } + + @Override + public void visitChildren(BiConsumer visitor) { + for (Entry child : entries) { + visitor.accept(child.getPath(), child.getValue()); + } + } + + protected int findChildIndexWithCommonPrefix(VfsRelativePath targetPath, CaseSensitivity caseSensitivity) { + return SearchUtil.binarySearch( + entries, + candidate -> targetPath.compareToFirstSegment(candidate.getPath(), caseSensitivity) + ); + } + + @Override + public ChildMap invalidate(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, InvalidationHandler handler) { + int childIndex = findChildIndexWithCommonPrefix(targetPath, caseSensitivity); + if (childIndex >= 0) { + Entry entry = entries.get(childIndex); + String childPath = entry.getPath(); + return entry.withNode(targetPath, caseSensitivity, new AbstractInvalidateChildHandler(handler) { + + @SuppressWarnings("unchecked") + @Override + public AbstractListChildMap getChildMap() { + return (AbstractListChildMap) AbstractListChildMap.this; + } + + @Override + public ChildMap withReplacedChild(RESULT newChild) { + return withReplacedChild(childPath, newChild); + } + + @Override + public ChildMap withReplacedChild(String newChildPath, RESULT newChild) { + return getChildMap().withReplacedChild(childIndex, newChildPath, newChild); + } + + @Override + public ChildMap withRemovedChild() { + return getChildMap().withRemovedChild(childIndex); + } + }); + } else { + handler.handleUnrelatedToAnyChild(); + @SuppressWarnings("unchecked") AbstractListChildMap castedThis = (AbstractListChildMap) this; + return castedThis; + } + } + + @Override + public ChildMap store(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, StoreHandler storeHandler) { + int childIndex = findChildIndexWithCommonPrefix(targetPath, caseSensitivity); + if (childIndex >= 0) { + return entries.get(childIndex).handlePath(targetPath, caseSensitivity, new AbstractStorePathRelationshipHandler(caseSensitivity, storeHandler) { + @Override + public ChildMap withReplacedChild(T newChild) { + return withReplacedChild(entries.get(childIndex).getPath(), newChild); + } + + @Override + public ChildMap withReplacedChild(String newChildPath, T newChild) { + return AbstractListChildMap.this.withReplacedChild(childIndex, newChildPath, newChild); + } + + @Override + public ChildMap withNewChild(String newChildPath, T newChild) { + return AbstractListChildMap.this.withNewChild(childIndex, newChildPath, newChild); + } + }); + } else { + T newChild = storeHandler.createChild(); + return withNewChild(-childIndex - 1, targetPath.toString(), newChild); + } + } + + protected ChildMap withNewChild(int insertBefore, String path, T newChild) { + List> newChildren = new ArrayList<>(entries); + newChildren.add(insertBefore, new Entry<>(path, newChild)); + return ChildMapFactory.childMapFromSorted(newChildren); + } + + protected ChildMap withReplacedChild(int childIndex, String newPath, T newChild) { + Entry oldEntry = entries.get(childIndex); + if (oldEntry.getPath().equals(newPath) && oldEntry.getValue().equals(newChild)) { + return this; + } + List> newChildren = new ArrayList<>(entries); + newChildren.set(childIndex, new Entry<>(newPath, newChild)); + return ChildMapFactory.childMapFromSorted(newChildren); + } + + protected ChildMap withRemovedChild(int childIndex) { + List> newChildren = new ArrayList<>(entries); + newChildren.remove(childIndex); + return ChildMapFactory.childMapFromSorted(newChildren); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + AbstractListChildMap that = (AbstractListChildMap) o; + + return entries.equals(that.entries); + } + + @Override + public int hashCode() { + return entries.hashCode(); + } + + @Override + public String toString() { + return entries.toString(); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/AbstractStorePathRelationshipHandler.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/AbstractStorePathRelationshipHandler.java new file mode 100644 index 00000000..c0798b8a --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/AbstractStorePathRelationshipHandler.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +public abstract class AbstractStorePathRelationshipHandler implements ChildMap.Entry.PathRelationshipHandler, T> { + + private final CaseSensitivity caseSensitivity; + private final ChildMap.StoreHandler handler; + + public AbstractStorePathRelationshipHandler(CaseSensitivity caseSensitivity, ChildMap.StoreHandler handler) { + this.caseSensitivity = caseSensitivity; + this.handler = handler; + } + + public abstract ChildMap withReplacedChild(T newChild); + + public abstract ChildMap withReplacedChild(String newChildPath, T newChild); + + public abstract ChildMap withNewChild(String newChildPath, T newChild); + + @Override + public ChildMap handleAsDescendantOfChild(VfsRelativePath targetPath, String childPath, T child) { + T newChild = handler.handleAsDescendantOfChild(targetPath.fromChild(childPath), child); + return withReplacedChild(newChild); + } + + @Override + public ChildMap handleAsAncestorOfChild(VfsRelativePath targetPath, String childPath, T child) { + T newChild = handler.handleAsAncestorOfChild(childPath, child); + return withReplacedChild(targetPath.getAsString(), newChild); + } + + @Override + public ChildMap handleExactMatchWithChild(VfsRelativePath targetPath, String childPath, T child) { + T newChild = handler.mergeWithExisting(child); + return withReplacedChild(newChild); + } + + @Override + public ChildMap handleSiblingOfChild(VfsRelativePath targetPath, String childPath, T child, int commonPrefixLength) { + String commonPrefix = childPath.substring(0, commonPrefixLength); + String newChildPath = childPath.substring(commonPrefixLength + 1); + ChildMap.Entry newChild = new ChildMap.Entry<>(newChildPath, child); + String siblingPath = targetPath.suffixStartingFrom(commonPrefixLength + 1).getAsString(); + ChildMap.Entry sibling = new ChildMap.Entry<>(siblingPath, handler.createChild()); + ChildMap newChildren = ChildMapFactory.childMap(caseSensitivity, newChild, sibling); + return withReplacedChild(commonPrefix, handler.createNodeFromChildren(newChildren)); + } + + @Override + public ChildMap handleUnrelatedToAnyChild(VfsRelativePath targetPath) { + String path = targetPath.getAsString(); + T newNode = handler.createChild(); + return withNewChild(path, newNode); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/CaseSensitivity.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/CaseSensitivity.java new file mode 100644 index 00000000..41ff1b9a --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/CaseSensitivity.java @@ -0,0 +1,28 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +/** + * The case sensitivity of a file system. + * + * Note that the method for actually comparing paths with a case sensitivity are in {@link PathUtil} instead of being on this enum, + * since it seems that the methods can be better inlined by the JIT compiler if they are static. + */ +public enum CaseSensitivity { + CASE_SENSITIVE, + CASE_INSENSITIVE +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ChildMap.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ChildMap.java new file mode 100644 index 00000000..15eae95d --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ChildMap.java @@ -0,0 +1,153 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import java.util.List; +import java.util.Optional; +import java.util.function.BiConsumer; + +public interface ChildMap { + + boolean isEmpty(); + + List values(); + + List> entries(); + + void visitChildren(BiConsumer visitor); + + RESULT withNode(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, NodeHandler handler); + + interface NodeHandler { + RESULT handleAsDescendantOfChild(VfsRelativePath pathInChild, T child); + RESULT handleAsAncestorOfChild(String childPath, T child); + RESULT handleExactMatchWithChild(T child); + RESULT handleUnrelatedToAnyChild(); + } + + ChildMap invalidate(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, InvalidationHandler handler); + + interface InvalidationHandler { + Optional handleAsDescendantOfChild(VfsRelativePath pathInChild, T child); + void handleAsAncestorOfChild(String childPath, T child); + void handleExactMatchWithChild(T child); + void handleUnrelatedToAnyChild(); + } + + ChildMap store(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, StoreHandler storeHandler); + + interface StoreHandler { + T handleAsDescendantOfChild(VfsRelativePath pathInChild, T child); + T handleAsAncestorOfChild(String childPath, T child); + T mergeWithExisting(T child); + T createChild(); + T createNodeFromChildren(ChildMap children); + } + + class Entry { + private final String path; + private final T value; + + public Entry(String path, T value) { + this.path = path; + this.value = value; + } + + public RESULT withNode(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, NodeHandler handler) { + return handleAncestorDescendantOrExactMatch(targetPath, caseSensitivity, handler) + .orElseGet(handler::handleUnrelatedToAnyChild); + } + + public Optional handleAncestorDescendantOrExactMatch(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, NodeHandler handler) { + if (targetPath.hasPrefix(path, caseSensitivity)) { + if (targetPath.length() == path.length()) { + return Optional.of(handler.handleExactMatchWithChild(value)); + } else { + return Optional.of(handler.handleAsDescendantOfChild(targetPath.fromChild(path), value)); + } + } else if (targetPath.length() < path.length() && targetPath.isPrefixOf(path, caseSensitivity)) { + return Optional.of(handler.handleAsAncestorOfChild(path, value)); + } + return Optional.empty(); + } + + public RESULT handlePath(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, PathRelationshipHandler handler) { + int pathToParentLength = path.length(); + int targetPathLength = targetPath.length(); + int maxPos = Math.min(pathToParentLength, targetPathLength); + int commonPrefixLength = targetPath.lengthOfCommonPrefix(path, caseSensitivity); + if (commonPrefixLength == maxPos) { + if (pathToParentLength > targetPathLength) { + return handler.handleAsAncestorOfChild(targetPath, path, value); + } + if (pathToParentLength == targetPathLength) { + return handler.handleExactMatchWithChild(targetPath, path, value); + } + return handler.handleAsDescendantOfChild(targetPath, path, value); + } + if (commonPrefixLength == 0) { + return handler.handleUnrelatedToAnyChild(targetPath); + } + return handler.handleSiblingOfChild(targetPath, path, value, commonPrefixLength); + } + + public interface PathRelationshipHandler { + RESULT handleAsDescendantOfChild(VfsRelativePath targetPath, String childPath, T child); + RESULT handleAsAncestorOfChild(VfsRelativePath targetPath, String childPath, T child); + RESULT handleExactMatchWithChild(VfsRelativePath targetPath, String childPath, T child); + RESULT handleSiblingOfChild(VfsRelativePath targetPath, String childPath, T child, int commonPrefixLength); + RESULT handleUnrelatedToAnyChild(VfsRelativePath targetPath); + } + + public String getPath() { + return path; + } + + public T getValue() { + return value; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + Entry entry = (Entry) o; + + if (!path.equals(entry.path)) { + return false; + } + return value.equals(entry.value); + } + + @Override + public int hashCode() { + int result = path.hashCode(); + result = 31 * result + value.hashCode(); + return result; + } + + @Override + public String toString() { + return "Entry{" + path + " : " + value + '}'; + } + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ChildMapFactory.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ChildMapFactory.java new file mode 100644 index 00000000..61cded95 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ChildMapFactory.java @@ -0,0 +1,62 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; + +public class ChildMapFactory { + /** + * If a node has fewer children, we use a linear search for the child. + * We use this limit since {@link VfsRelativePath#compareToFirstSegment(String, CaseSensitivity)} + * is about twice as slow as {@link VfsRelativePath#hasPrefix(String, CaseSensitivity)}, + * so comparing the searched path to all of the children is actually faster than doing a binary search. + */ + private static final int MINIMUM_CHILD_COUNT_FOR_BINARY_SEARCH = 10; + + public static ChildMap childMap(CaseSensitivity caseSensitivity, Collection> entries) { + List> sortedEntries = new ArrayList<>(entries); + sortedEntries.sort(Comparator.comparing(ChildMap.Entry::getPath, PathUtil.getPathComparator(caseSensitivity))); + return childMapFromSorted(sortedEntries); + } + + public static ChildMap childMapFromSorted(List> sortedEntries) { + int size = sortedEntries.size(); + switch (size) { + case 0: + return EmptyChildMap.getInstance(); + case 1: + return new SingletonChildMap<>(sortedEntries.get(0)); + default: + return (size < MINIMUM_CHILD_COUNT_FOR_BINARY_SEARCH) + ? new MediumChildMap<>(sortedEntries) + : new LargeChildMap<>(sortedEntries); + } + } + + static ChildMap childMap(CaseSensitivity caseSensitivity, ChildMap.Entry entry1, ChildMap.Entry entry2) { + int compared = PathUtil.getPathComparator(caseSensitivity).compare(entry1.getPath(), entry2.getPath()); + List> sortedEntries = compared < 0 + ? ImmutableList.of(entry1, entry2) + : ImmutableList.of(entry2, entry1); + return childMapFromSorted(sortedEntries); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/CompositeFileSystemSnapshot.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/CompositeFileSystemSnapshot.java new file mode 100644 index 00000000..13f4f52c --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/CompositeFileSystemSnapshot.java @@ -0,0 +1,86 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import com.google.common.collect.ImmutableList; + +import java.util.Collection; +import java.util.List; + +public class CompositeFileSystemSnapshot implements FileSystemSnapshot { + private final ImmutableList snapshots; + + private CompositeFileSystemSnapshot(Collection snapshots) { + this.snapshots = ImmutableList.copyOf(snapshots); + } + + public static FileSystemSnapshot of(List snapshots) { + switch (snapshots.size()) { + case 0: + return EMPTY; + case 1: + return snapshots.get(0); + default: + return new CompositeFileSystemSnapshot(snapshots); + } + } + + @Override + public SnapshotVisitResult accept(FileSystemSnapshotHierarchyVisitor visitor) { + for (FileSystemSnapshot snapshot : snapshots) { + SnapshotVisitResult result = snapshot.accept(visitor); + if (result == SnapshotVisitResult.TERMINATE) { + return SnapshotVisitResult.TERMINATE; + } + } + return SnapshotVisitResult.CONTINUE; + } + + @Override + public SnapshotVisitResult accept(RelativePathTracker pathTracker, RelativePathTrackingFileSystemSnapshotHierarchyVisitor visitor) { + for (FileSystemSnapshot snapshot : snapshots) { + SnapshotVisitResult result = snapshot.accept(pathTracker, visitor); + if (result == SnapshotVisitResult.TERMINATE) { + return SnapshotVisitResult.TERMINATE; + } + } + return SnapshotVisitResult.CONTINUE; + } + + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + CompositeFileSystemSnapshot that = (CompositeFileSystemSnapshot) o; + + return snapshots.equals(that.snapshots); + } + + @Override + public int hashCode() { + return snapshots.hashCode(); + } + + @Override + public String toString() { + return snapshots.toString(); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/DirectorySnapshot.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/DirectorySnapshot.java new file mode 100644 index 00000000..c395b7cc --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/DirectorySnapshot.java @@ -0,0 +1,197 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import com.google.common.annotations.VisibleForTesting; +import org.gradle.internal.file.FileMetadata.AccessType; +import org.gradle.internal.file.FileType; +import org.gradle.internal.hash.HashCode; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.gradle.internal.snapshot.ChildMapFactory.childMapFromSorted; +import static org.gradle.internal.snapshot.SnapshotVisitResult.CONTINUE; + +/** + * A snapshot of an existing directory hierarchy. + * + * Includes snapshots of any child element and the Merkle tree hash. + */ +public class DirectorySnapshot extends AbstractFileSystemLocationSnapshot { + private final ChildMap children; + private final HashCode contentHash; + + public DirectorySnapshot(String absolutePath, String name, AccessType accessType, HashCode contentHash, List children) { + this(absolutePath, name, accessType, contentHash, childMapFromSorted(children.stream() + .map(it -> new ChildMap.Entry<>(it.getName(), it)) + .collect(Collectors.toList()))); + } + + public DirectorySnapshot(String absolutePath, String name, AccessType accessType, HashCode contentHash, ChildMap children) { + super(absolutePath, name, accessType); + this.contentHash = contentHash; + this.children = children; + } + + @Override + public HashCode getHash() { + return contentHash; + } + + @Override + public FileType getType() { + return FileType.Directory; + } + + @Override + public boolean isContentAndMetadataUpToDate(FileSystemLocationSnapshot other) { + return isContentUpToDate(other); + } + + @Override + public boolean isContentUpToDate(FileSystemLocationSnapshot other) { + return other instanceof DirectorySnapshot; + } + + @Override + public SnapshotVisitResult accept(FileSystemSnapshotHierarchyVisitor visitor) { + SnapshotVisitResult result = visitor.visitEntry(this); + switch (result) { + case CONTINUE: + visitor.enterDirectory(this); + children.visitChildren((name, child) -> child.accept(visitor)); + visitor.leaveDirectory(this); + return CONTINUE; + case SKIP_SUBTREE: + return CONTINUE; + case TERMINATE: + return SnapshotVisitResult.TERMINATE; + default: + throw new AssertionError(); + } + } + + @Override + public SnapshotVisitResult accept(RelativePathTracker pathTracker, RelativePathTrackingFileSystemSnapshotHierarchyVisitor visitor) { + pathTracker.enter(getName()); + try { + SnapshotVisitResult result = visitor.visitEntry(this, pathTracker); + switch (result) { + case CONTINUE: + visitor.enterDirectory(this, pathTracker); + children.visitChildren((name, child) -> child.accept(pathTracker, visitor)); + visitor.leaveDirectory(this, pathTracker); + return CONTINUE; + case SKIP_SUBTREE: + return CONTINUE; + case TERMINATE: + return SnapshotVisitResult.TERMINATE; + default: + throw new AssertionError(); + } + } finally { + pathTracker.leave(); + } + } + + @Override + public void accept(FileSystemLocationSnapshotVisitor visitor) { + visitor.visitDirectory(this); + } + + @Override + public T accept(FileSystemLocationSnapshotTransformer transformer) { + return transformer.visitDirectory(this); + } + + @VisibleForTesting + public List getChildren() { + return children.values(); + } + + @Override + protected Optional getChildSnapshot(VfsRelativePath targetPath, CaseSensitivity caseSensitivity) { + return Optional.of( + SnapshotUtil.getMetadataFromChildren(children, targetPath, caseSensitivity, Optional::empty) + .orElseGet(() -> missingSnapshotForAbsolutePath(targetPath.getAbsolutePath())) + ); + } + + @Override + protected ReadOnlyFileSystemNode getChildNode(VfsRelativePath targetPath, CaseSensitivity caseSensitivity) { + ReadOnlyFileSystemNode childNode = SnapshotUtil.getChild(children, targetPath, caseSensitivity); + return childNode == ReadOnlyFileSystemNode.EMPTY + ? missingSnapshotForAbsolutePath(targetPath.getAbsolutePath()) + : childNode; + } + + @Override + public Optional invalidate(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, SnapshotHierarchy.NodeDiffListener diffListener) { + ChildMap newChildren = children.invalidate(targetPath, caseSensitivity, new ChildMap.InvalidationHandler() { + @Override + public Optional handleAsDescendantOfChild(VfsRelativePath pathInChild, FileSystemLocationSnapshot child) { + diffListener.nodeRemoved(DirectorySnapshot.this); + Optional invalidated = child.invalidate(pathInChild, caseSensitivity, new SnapshotHierarchy.NodeDiffListener() { + @Override + public void nodeRemoved(FileSystemNode node) { + // the parent already has been removed. No children need to be removed. + } + + @Override + public void nodeAdded(FileSystemNode node) { + diffListener.nodeAdded(node); + } + }); + children.visitChildren((__, existingChild) -> { + if (existingChild != child) { + diffListener.nodeAdded(existingChild); + } + }); + return invalidated; + } + + @Override + public void handleAsAncestorOfChild(String childPath, FileSystemLocationSnapshot child) { + throw new IllegalStateException("Can't have an ancestor of a single path element"); + } + + @Override + public void handleExactMatchWithChild(FileSystemLocationSnapshot child) { + diffListener.nodeRemoved(DirectorySnapshot.this); + children.visitChildren((__, existingChild) -> { + if (existingChild != child) { + diffListener.nodeAdded(existingChild); + } + }); + } + + @Override + public void handleUnrelatedToAnyChild() { + diffListener.nodeRemoved(DirectorySnapshot.this); + children.visitChildren((__, child) -> diffListener.nodeAdded(child)); + } + }); + return Optional.of(new PartialDirectoryNode(newChildren)); + } + + @Override + public String toString() { + return String.format("%s@%s/%s(%s)", super.toString(), contentHash, getName(), children); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/EmptyChildMap.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/EmptyChildMap.java new file mode 100644 index 00000000..6eeb1ddd --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/EmptyChildMap.java @@ -0,0 +1,73 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import java.util.Collections; +import java.util.List; +import java.util.function.BiConsumer; + +public class EmptyChildMap implements ChildMap { + private static final EmptyChildMap INSTANCE = new EmptyChildMap<>(); + + @SuppressWarnings("unchecked") + public static EmptyChildMap getInstance() { + return (EmptyChildMap) INSTANCE; + } + + private EmptyChildMap() { + } + + @Override + public R withNode(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, NodeHandler handler) { + return handler.handleUnrelatedToAnyChild(); + } + + @Override + public ChildMap invalidate(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, InvalidationHandler handler) { + handler.handleUnrelatedToAnyChild(); + return getInstance(); + } + + @Override + public ChildMap store(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, StoreHandler storeHandler) { + return new SingletonChildMap<>(targetPath.getAsString(), storeHandler.createChild()); + } + + @Override + public void visitChildren(BiConsumer visitor) { + } + + @Override + public boolean isEmpty() { + return true; + } + + @Override + public List values() { + return Collections.emptyList(); + } + + @Override + public List> entries() { + return Collections.emptyList(); + } + + @Override + public String toString() { + return ""; + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSnapshottingException.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSnapshottingException.java new file mode 100644 index 00000000..c4563327 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSnapshottingException.java @@ -0,0 +1,27 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +public class FileSnapshottingException extends RuntimeException { + public FileSnapshottingException(String message) { + super(message); + } + + public FileSnapshottingException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSystemLeafSnapshot.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSystemLeafSnapshot.java new file mode 100644 index 00000000..4949a4c9 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSystemLeafSnapshot.java @@ -0,0 +1,37 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +/** + * The snapshot of a leaf element in the file system that can have no children of its own. + */ +public interface FileSystemLeafSnapshot extends FileSystemLocationSnapshot { + @Override + default SnapshotVisitResult accept(FileSystemSnapshotHierarchyVisitor visitor) { + return visitor.visitEntry(this); + } + + @Override + default SnapshotVisitResult accept(RelativePathTracker pathTracker, RelativePathTrackingFileSystemSnapshotHierarchyVisitor visitor) { + pathTracker.enter(getName()); + try { + return visitor.visitEntry(this, pathTracker); + } finally { + pathTracker.leave(); + } + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSystemLocationSnapshot.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSystemLocationSnapshot.java new file mode 100644 index 00000000..ed9341a2 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSystemLocationSnapshot.java @@ -0,0 +1,95 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import org.gradle.internal.file.FileMetadata; +import org.gradle.internal.hash.HashCode; + +import java.util.Comparator; + +/** + * A snapshot of a single location on the file system. + * + * We know everything about this snapshot, including children and Merkle hash. + * + * The snapshot can be a snapshot of a regular file or of a whole directory tree. + * The file at the location is not required to exist (see {@link MissingFileSnapshot}. + */ +public interface FileSystemLocationSnapshot extends FileSystemSnapshot, FileSystemNode, MetadataSnapshot { + + /** + * The comparator of direct children of a file system location. + * + * The comparison is stable with respect to case sensitivity, so the order of the children is stable across operating systems. + */ + Comparator BY_NAME = Comparator.comparing(FileSystemLocationSnapshot::getName, PathUtil::compareFileNames); + + /** + * The file name. + */ + String getName(); + + /** + * The absolute path of the file. + */ + String getAbsolutePath(); + + /** + * The hash of the snapshot. + * + * This makes it possible to uniquely identify the snapshot. + *
+ *
Directories
+ *
The combined hash of the children, calculated by appending the name and the hash of each child to a hasher.
+ *
Regular Files
+ *
The hash of the content of the file.
+ *
Missing files
+ *
A special signature denoting a missing file.
+ *
+ */ + HashCode getHash(); + + /** + * Whether the content and the metadata (modification date) of the current snapshot is the same as for the given one. + */ + boolean isContentAndMetadataUpToDate(FileSystemLocationSnapshot other); + + /** + * Whether the content of the current snapshot is the same as for the given one. + */ + boolean isContentUpToDate(FileSystemLocationSnapshot other); + + /** + * Whether the file system location represented by this snapshot is a symlink or not. + */ + FileMetadata.AccessType getAccessType(); + + void accept(FileSystemLocationSnapshotVisitor visitor); + T accept(FileSystemLocationSnapshotTransformer transformer); + + interface FileSystemLocationSnapshotVisitor { + default void visitDirectory(DirectorySnapshot directorySnapshot) {}; + default void visitRegularFile(RegularFileSnapshot fileSnapshot) {}; + default void visitMissing(MissingFileSnapshot missingSnapshot) {}; + } + + interface FileSystemLocationSnapshotTransformer { + T visitDirectory(DirectorySnapshot directorySnapshot); + T visitRegularFile(RegularFileSnapshot fileSnapshot); + T visitMissing(MissingFileSnapshot missingSnapshot); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSystemNode.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSystemNode.java new file mode 100644 index 00000000..2ce8945c --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSystemNode.java @@ -0,0 +1,37 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import java.util.Optional; + +/** + * Any snapshot in the tree of the virtual file system. + */ +public interface FileSystemNode extends ReadOnlyFileSystemNode { + + /** + * Stores information to the virtual file system that we have learned about. + * + * Complete information, like {@link FileSystemLocationSnapshot}s, are not touched nor replaced. + */ + FileSystemNode store(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, MetadataSnapshot snapshot, SnapshotHierarchy.NodeDiffListener diffListener); + + /** + * Invalidates part of the node. + */ + Optional invalidate(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, SnapshotHierarchy.NodeDiffListener diffListener); +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSystemSnapshot.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSystemSnapshot.java new file mode 100644 index 00000000..c91fb185 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSystemSnapshot.java @@ -0,0 +1,51 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +/** + * A snapshot of a part of the file system. + */ +public interface FileSystemSnapshot { + /** + * An empty snapshot. + */ + FileSystemSnapshot EMPTY = new FileSystemSnapshot() { + @Override + public SnapshotVisitResult accept(FileSystemSnapshotHierarchyVisitor visitor) { + return SnapshotVisitResult.CONTINUE; + } + + @Override + public SnapshotVisitResult accept(RelativePathTracker pathTracker, RelativePathTrackingFileSystemSnapshotHierarchyVisitor visitor) { + return SnapshotVisitResult.CONTINUE; + } + }; + + /** + * Walks the whole hierarchy represented by this snapshot. + * + * The walk is depth first. + */ + SnapshotVisitResult accept(FileSystemSnapshotHierarchyVisitor visitor); + + /** + * Walks the whole hierarchy represented by this snapshot. + * + * The walk is depth first. + */ + SnapshotVisitResult accept(RelativePathTracker pathTracker, RelativePathTrackingFileSystemSnapshotHierarchyVisitor visitor); +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSystemSnapshotHierarchyVisitor.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSystemSnapshotHierarchyVisitor.java new file mode 100644 index 00000000..cad4ad0f --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/FileSystemSnapshotHierarchyVisitor.java @@ -0,0 +1,41 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +/** + * Visitor for {@link FileSystemSnapshot}. + */ +public interface FileSystemSnapshotHierarchyVisitor { + + /** + * Called before visiting the contents of a directory. + */ + default void enterDirectory(DirectorySnapshot directorySnapshot) {} + + /** + * Called for each regular file/directory/missing/unavailable file. + * + * @return how to continue visiting the rest of the snapshot hierarchy. + */ + SnapshotVisitResult visitEntry(FileSystemLocationSnapshot snapshot); + + /** + * Called after all entries in the directory has been visited. + */ + default void leaveDirectory(DirectorySnapshot directorySnapshot) {} + +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/LargeChildMap.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/LargeChildMap.java new file mode 100644 index 00000000..bd53c2ed --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/LargeChildMap.java @@ -0,0 +1,36 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import java.util.List; + +public class LargeChildMap extends AbstractListChildMap { + + public LargeChildMap(List> children) { + super(children); + } + + @Override + public R withNode(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, NodeHandler handler) { + int childIndexWithCommonPrefix = findChildIndexWithCommonPrefix(targetPath, caseSensitivity); + if (childIndexWithCommonPrefix >= 0) { + Entry entry = entries.get(childIndexWithCommonPrefix); + return entry.withNode(targetPath, caseSensitivity, handler); + } + return handler.handleUnrelatedToAnyChild(); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/MediumChildMap.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/MediumChildMap.java new file mode 100644 index 00000000..11bc99b9 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/MediumChildMap.java @@ -0,0 +1,37 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import java.util.List; +import java.util.Optional; + +public class MediumChildMap extends AbstractListChildMap { + protected MediumChildMap(List> children) { + super(children); + } + + @Override + public RESULT withNode(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, NodeHandler handler) { + for (Entry entry : entries) { + Optional ancestorDescendantOrExactMatchResult = entry.handleAncestorDescendantOrExactMatch(targetPath, caseSensitivity, handler); + if (ancestorDescendantOrExactMatchResult.isPresent()) { + return ancestorDescendantOrExactMatchResult.get(); + } + } + return handler.handleUnrelatedToAnyChild(); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/MerkleDirectorySnapshotBuilder.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/MerkleDirectorySnapshotBuilder.java new file mode 100644 index 00000000..f7bef748 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/MerkleDirectorySnapshotBuilder.java @@ -0,0 +1,132 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import org.gradle.internal.file.FileMetadata.AccessType; +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.hash.Hasher; +import org.gradle.internal.hash.Hashing; + +import javax.annotation.Nullable; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.List; + +import static org.gradle.internal.snapshot.MerkleDirectorySnapshotBuilder.EmptyDirectoryHandlingStrategy.EXCLUDE_EMPTY_DIRS; + +public class MerkleDirectorySnapshotBuilder { + private static final HashCode DIR_SIGNATURE = Hashing.signature("DIR"); + + private final Deque directoryStack = new ArrayDeque<>(); + private final boolean sortingRequired; + private FileSystemLocationSnapshot result; + + public static MerkleDirectorySnapshotBuilder sortingRequired() { + return new MerkleDirectorySnapshotBuilder(true); + } + + public static MerkleDirectorySnapshotBuilder noSortingRequired() { + return new MerkleDirectorySnapshotBuilder(false); + } + + private MerkleDirectorySnapshotBuilder(boolean sortingRequired) { + this.sortingRequired = sortingRequired; + } + + public void enterDirectory(DirectorySnapshot directorySnapshot, EmptyDirectoryHandlingStrategy emptyDirectoryHandlingStrategy) { + enterDirectory(directorySnapshot.getAccessType(), directorySnapshot.getAbsolutePath(), directorySnapshot.getName(), emptyDirectoryHandlingStrategy); + } + + public void enterDirectory(AccessType accessType, String absolutePath, String name, EmptyDirectoryHandlingStrategy emptyDirectoryHandlingStrategy) { + directoryStack.addLast(new Directory(accessType, absolutePath, name, emptyDirectoryHandlingStrategy)); + } + + public void visitLeafElement(FileSystemLeafSnapshot snapshot) { + collectEntry(snapshot); + } + + public void visitDirectory(DirectorySnapshot directorySnapshot) { + collectEntry(directorySnapshot); + } + + public boolean leaveDirectory() { + FileSystemLocationSnapshot snapshot = directoryStack.removeLast().fold(); + if (snapshot == null) { + return false; + } + collectEntry(snapshot); + return true; + } + + private void collectEntry(FileSystemLocationSnapshot snapshot) { + Directory directory = directoryStack.peekLast(); + if (directory != null) { + directory.collectEntry(snapshot); + } else { + assert result == null; + result = snapshot; + } + } + + @Nullable + public FileSystemLocationSnapshot getResult() { + return result; + } + + public enum EmptyDirectoryHandlingStrategy { + INCLUDE_EMPTY_DIRS, + EXCLUDE_EMPTY_DIRS + } + + private class Directory { + private final AccessType accessType; + private final String absolutePath; + private final String name; + private final List children; + private final EmptyDirectoryHandlingStrategy emptyDirectoryHandlingStrategy; + + public Directory(AccessType accessType, String absolutePath, String name, EmptyDirectoryHandlingStrategy emptyDirectoryHandlingStrategy) { + this.accessType = accessType; + this.absolutePath = absolutePath; + this.name = name; + this.children = new ArrayList<>(); + this.emptyDirectoryHandlingStrategy = emptyDirectoryHandlingStrategy; + } + + public void collectEntry(FileSystemLocationSnapshot snapshot) { + children.add(snapshot); + } + + @Nullable + public DirectorySnapshot fold() { + if (emptyDirectoryHandlingStrategy == EXCLUDE_EMPTY_DIRS && children.isEmpty()) { + return null; + } + if (sortingRequired) { + children.sort(FileSystemLocationSnapshot.BY_NAME); + } + Hasher hasher = Hashing.newHasher(); + hasher.putHash(DIR_SIGNATURE); + for (FileSystemLocationSnapshot child : children) { + hasher.putString(child.getName()); + hasher.putHash(child.getHash()); + } + return new DirectorySnapshot(absolutePath, name, accessType, hasher.hash(), children); + } + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/MetadataSnapshot.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/MetadataSnapshot.java new file mode 100644 index 00000000..c2a43415 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/MetadataSnapshot.java @@ -0,0 +1,44 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import org.gradle.internal.file.FileType; + +/** + * A snapshot where we know the metadata (i.e. the type). + */ +public interface MetadataSnapshot { + + MetadataSnapshot DIRECTORY = new MetadataSnapshot() { + @Override + public FileType getType() { + return FileType.Directory; + } + + @Override + public FileSystemNode asFileSystemNode() { + return PartialDirectoryNode.withoutKnownChildren(); + } + }; + + /** + * The type of the file. + */ + FileType getType(); + + FileSystemNode asFileSystemNode(); +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/MissingFileSnapshot.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/MissingFileSnapshot.java new file mode 100644 index 00000000..a83fdb51 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/MissingFileSnapshot.java @@ -0,0 +1,79 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import org.gradle.internal.file.FileMetadata.AccessType; +import org.gradle.internal.file.FileType; +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.hash.Hashing; + +import java.util.Optional; + +/** + * A snapshot of a missing file or a broken symbolic link or a named pipe. + */ +public class MissingFileSnapshot extends AbstractFileSystemLocationSnapshot implements FileSystemLeafSnapshot { + private static final HashCode SIGNATURE = Hashing.signature(MissingFileSnapshot.class); + + public MissingFileSnapshot(String absolutePath, String name, AccessType accessType) { + super(absolutePath, name, accessType); + } + + public MissingFileSnapshot(String absolutePath, AccessType accessType) { + this(absolutePath, PathUtil.getFileName(absolutePath), accessType); + } + + @Override + public FileType getType() { + return FileType.Missing; + } + + @Override + public HashCode getHash() { + return SIGNATURE; + } + + @Override + public boolean isContentAndMetadataUpToDate(FileSystemLocationSnapshot other) { + return isContentUpToDate(other); + } + + @Override + public boolean isContentUpToDate(FileSystemLocationSnapshot other) { + return other instanceof MissingFileSnapshot; + } + + @Override + public void accept(FileSystemLocationSnapshotVisitor visitor) { + visitor.visitMissing(this); + } + + @Override + public T accept(FileSystemLocationSnapshotTransformer transformer) { + return transformer.visitMissing(this); + } + + public Optional invalidate(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, SnapshotHierarchy.NodeDiffListener diffListener) { + diffListener.nodeRemoved(this); + return Optional.empty(); + } + + @Override + public String toString() { + return String.format("%s/%s", super.toString(), getName()); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/PartialDirectoryNode.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/PartialDirectoryNode.java new file mode 100644 index 00000000..508b417e --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/PartialDirectoryNode.java @@ -0,0 +1,55 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import java.util.Optional; + +/** + * An incomplete snapshot of an existing directory. + * + * May include some of its children. + */ +public class PartialDirectoryNode extends AbstractIncompleteFileSystemNode { + + public static PartialDirectoryNode withoutKnownChildren() { + return new PartialDirectoryNode(EmptyChildMap.getInstance()); + } + + public PartialDirectoryNode(ChildMap children) { + super(children); + } + + @Override + protected FileSystemNode withIncompleteChildren(ChildMap newChildren) { + return new PartialDirectoryNode(newChildren); + } + + @Override + protected Optional withAllChildrenRemoved() { + return Optional.of(children.isEmpty() ? this : withoutKnownChildren()); + } + + @Override + public Optional getSnapshot() { + return Optional.of(MetadataSnapshot.DIRECTORY); + } + + @Override + protected FileSystemNode withIncompleteChildren() { + return this; + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/PathUtil.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/PathUtil.java new file mode 100644 index 00000000..86d209d6 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/PathUtil.java @@ -0,0 +1,214 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import com.google.common.annotations.VisibleForTesting; + +import java.io.File; +import java.util.Comparator; + +import static org.gradle.internal.snapshot.CaseSensitivity.CASE_INSENSITIVE; +import static org.gradle.internal.snapshot.CaseSensitivity.CASE_SENSITIVE; + +/** + * Methods for dealing with paths on the file system. + * + * There are methods for checking equality and for comparing two paths. + * All methods for equality and comparing need to be called with the correct case-sensitivity according to the underlying file system. + * + * A segment of a path is the part between two file separators. + * For example, the path some/long/path has the segments some, long and path. + * + * For comparing, a list of paths is sorted in the same order on a case-insensitive and a case-sensitive file system. + * We do this so the order of the children of directory snapshots is stable across builds. + * + * The order is as follows: + * - The comparison is per segment of the path. + * - If the segments are different with respect to case-insensitive comparison, the result from case-insensitive comparison is used. + * - If one segment starts with the other segment comparing case-insensitive, then the shorter segment is smaller. + * - Finally, if both segments are the same ignoring case and have the same length, the case-sensitive comparison is used. + * + * For all methods operating on a list of paths, the paths must not start with a common segment. + * For example, ["some", "some1/other", "other/foo"] is allowed, but ["some/foo", "some/bar", "other/foo"] is not. + */ +public class PathUtil { + + /** + * The Unix separator character. + */ + private static final char UNIX_SEPARATOR = '/'; + + /** + * The Windows separator character. + */ + private static final char WINDOWS_SEPARATOR = '\\'; + + /** + * The system separator character. + */ + private static final char SYSTEM_SEPARATOR = File.separatorChar; + + private static final boolean IS_WINDOWS_SEPARATOR = SYSTEM_SEPARATOR == WINDOWS_SEPARATOR; + + /** + * The separator character that is the opposite of the system separator. + */ + private static final char OTHER_SEPARATOR = IS_WINDOWS_SEPARATOR ? UNIX_SEPARATOR : WINDOWS_SEPARATOR; + + private static final Comparator CASE_SENSITIVE_COMPARATOR = (path1, path2) -> comparePaths(path1, path2, CASE_SENSITIVE); + private static final Comparator CASE_INSENSITIVE_COMPARATOR = (path1, path2) -> comparePaths(path1, path2, CASE_INSENSITIVE); + + /** + * Whether the given char is a file separator. + * Both Unix and Windows file separators are detected, no matter the current operating system. + */ + public static boolean isFileSeparator(char toCheck) { + return toCheck == SYSTEM_SEPARATOR || toCheck == OTHER_SEPARATOR; + } + + /** + * Compares two file names with the order defined here: {@link PathUtil}. + * + * File names do not contain file separators, so the methods on {@link String} can be used for the comparison. + */ + public static int compareFileNames(String name1, String name2) { + int caseInsensitiveComparison = name1.compareToIgnoreCase(name2); + return caseInsensitiveComparison != 0 + ? caseInsensitiveComparison + : name1.compareTo(name2); + } + + /** + * Returns a comparator for paths for the given case sensitivity. + * + * When the two paths are different ignoring the case, then the result of the comparison is the same for both comparators. + */ + public static Comparator getPathComparator(CaseSensitivity caseSensitivity) { + switch (caseSensitivity) { + case CASE_SENSITIVE: + return CASE_SENSITIVE_COMPARATOR; + case CASE_INSENSITIVE: + return CASE_INSENSITIVE_COMPARATOR; + default: + throw new AssertionError(); + } + } + + @VisibleForTesting + static int compareCharsIgnoringCase(char char1, char char2) { + if (char1 == char2) { + return 0; + } + return isFileSeparator(char1) + ? isFileSeparator(char2) + ? 0 + : -1 + : isFileSeparator(char2) + ? 1 + : compareDifferentCharsIgnoringCase(char1, char2); + } + + private static int compareDifferentCharsIgnoringCase(char char1, char char2) { + char insensitiveChar1 = Character.toUpperCase(char1); + char insensitiveChar2 = Character.toUpperCase(char2); + if (insensitiveChar1 != insensitiveChar2) { + insensitiveChar1 = Character.toLowerCase(insensitiveChar1); + insensitiveChar2 = Character.toLowerCase(insensitiveChar2); + if (insensitiveChar1 != insensitiveChar2) { + return Character.compare(insensitiveChar1, insensitiveChar2); + } + } + return 0; + } + + @VisibleForTesting + static int compareChars(char char1, char char2) { + if (char1 == char2) { + return 0; + } + return isFileSeparator(char1) + ? isFileSeparator(char2) + ? 0 + : -1 + : isFileSeparator(char2) + ? 1 + : Character.compare(char1, char2); + } + + @VisibleForTesting + static boolean equalChars(char char1, char char2, CaseSensitivity caseSensitivity) { + if (char1 == char2) { + return true; + } + if (isFileSeparator(char1) && isFileSeparator(char2)) { + return true; + } + if (caseSensitivity == CASE_SENSITIVE) { + return false; + } else { + return Character.toUpperCase(char1) == Character.toUpperCase(char2) || + Character.toLowerCase(char1) == Character.toLowerCase(char2); + } + } + + private static int comparePaths(String relativePath1, String relativePath2, CaseSensitivity caseSensitivity) { + int maxPos = Math.min(relativePath1.length(), relativePath2.length()); + int accumulatedValue = 0; + for (int pos = 0; pos < maxPos; pos++) { + char charInPath1 = relativePath1.charAt(pos); + char charInPath2 = relativePath2.charAt(pos); + int comparedChars = compareCharsIgnoringCase(charInPath1, charInPath2); + if (comparedChars != 0) { + return comparedChars; + } + accumulatedValue = computeCombinedCompare(accumulatedValue, charInPath1, charInPath2, caseSensitivity == CASE_SENSITIVE); + if (accumulatedValue != 0 && isFileSeparator(charInPath1)) { + return accumulatedValue; + } + } + int lengthCompare = Integer.compare(relativePath1.length(), relativePath2.length()); + return lengthCompare != 0 + ? lengthCompare + : accumulatedValue; + } + + private static int computeCombinedCompare(int previousCombinedValue, char charInPath1, char charInPath2, boolean caseSensitive) { + if (!caseSensitive) { + return 0; + } + return previousCombinedValue == 0 + ? compareChars(charInPath1, charInPath2) + : previousCombinedValue; + } + + public static String getFileName(String absolutePath) { + int lastSeparator = lastIndexOfSeparator(absolutePath); + return lastSeparator < 0 + ? absolutePath + : absolutePath.substring(lastSeparator + 1); + } + + private static int lastIndexOfSeparator(String absolutePath) { + for (int i = absolutePath.length() - 1; i >= 0; i--) { + char currentChar = absolutePath.charAt(i); + if (isFileSeparator(currentChar)) { + return i; + } + } + return -1; + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ReadOnlyFileSystemNode.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ReadOnlyFileSystemNode.java new file mode 100644 index 00000000..39337ae6 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ReadOnlyFileSystemNode.java @@ -0,0 +1,72 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import javax.annotation.Nullable; +import java.util.Optional; + +public interface ReadOnlyFileSystemNode { + ReadOnlyFileSystemNode EMPTY = new ReadOnlyFileSystemNode() { + @Override + public Optional getSnapshot(VfsRelativePath relativePath, CaseSensitivity caseSensitivity) { + return Optional.empty(); + } + + @Override + public boolean hasDescendants() { + return false; + } + + @Override + public ReadOnlyFileSystemNode getNode(VfsRelativePath relativePath, CaseSensitivity caseSensitivity) { + return EMPTY; + } + + @Override + public Optional getSnapshot() { + return Optional.empty(); + } + + @Override + public void accept(SnapshotHierarchy.SnapshotVisitor snapshotVisitor) { + } + }; + + /** + * Gets a snapshot from the current node with relative path filePath.substring(offset). + * + * When calling this method, the caller needs to make sure the the snapshot is a child of this node. + */ + Optional getSnapshot(VfsRelativePath relativePath, CaseSensitivity caseSensitivity); + + boolean hasDescendants(); + + ReadOnlyFileSystemNode getNode(VfsRelativePath relativePath, CaseSensitivity caseSensitivity); + + /** + * The snapshot information at this node. + * + * {@link Optional#empty()} if no information is available. + */ + Optional getSnapshot(); + + void accept(SnapshotHierarchy.SnapshotVisitor snapshotVisitor); + + interface NodeVisitor { + void visitNode(FileSystemNode node, @Nullable FileSystemNode parent); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/RegularFileSnapshot.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/RegularFileSnapshot.java new file mode 100644 index 00000000..08261abb --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/RegularFileSnapshot.java @@ -0,0 +1,88 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import org.gradle.internal.file.FileMetadata; +import org.gradle.internal.file.FileType; +import org.gradle.internal.hash.HashCode; + +import java.util.Optional; + +/** + * A snapshot of a regular file. + * + * The snapshot includes the content hash of the file and its metadata. + */ +public class RegularFileSnapshot extends AbstractFileSystemLocationSnapshot implements FileSystemLeafSnapshot { + private final HashCode contentHash; + private final FileMetadata metadata; + + public RegularFileSnapshot(String absolutePath, String name, HashCode contentHash, FileMetadata metadata) { + super(absolutePath, name, metadata.getAccessType()); + this.contentHash = contentHash; + this.metadata = metadata; + } + + @Override + public FileType getType() { + return FileType.RegularFile; + } + + @Override + public HashCode getHash() { + return contentHash; + } + + // Used by the Maven caching client. Do not remove + public FileMetadata getMetadata() { + return metadata; + } + + @Override + public boolean isContentAndMetadataUpToDate(FileSystemLocationSnapshot other) { + return isContentUpToDate(other) && metadata.equals(((RegularFileSnapshot) other).metadata); + } + + @Override + public boolean isContentUpToDate(FileSystemLocationSnapshot other) { + if (!(other instanceof RegularFileSnapshot)) { + return false; + } + return contentHash.equals(((RegularFileSnapshot) other).contentHash); + } + + @Override + public void accept(FileSystemLocationSnapshotVisitor visitor) { + visitor.visitRegularFile(this); + } + + @Override + public T accept(FileSystemLocationSnapshotTransformer transformer) { + return transformer.visitRegularFile(this); + } + + @Override + public Optional invalidate(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, SnapshotHierarchy.NodeDiffListener diffListener) { + diffListener.nodeRemoved(this); + return Optional.empty(); + } + + @Override + public String toString() { + return String.format("%s@%s/%s", super.toString(), getHash(), getName()); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/RelativePathTracker.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/RelativePathTracker.java new file mode 100644 index 00000000..2d00527c --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/RelativePathTracker.java @@ -0,0 +1,91 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import org.gradle.internal.RelativePathSupplier; + +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.Deque; +import java.util.Iterator; + +/** + * Tracks the relative path. Useful when visiting {@link FileSystemLocationSnapshot}s. + */ +public class RelativePathTracker implements RelativePathSupplier { + private final Deque segments = new ArrayDeque<>(); + private String rootName; + + public void enter(FileSystemLocationSnapshot snapshot) { + enter(snapshot.getName()); + } + + public void enter(String name) { + if (rootName == null) { + rootName = name; + } else { + segments.addLast(name); + } + } + + public String leave() { + String name = segments.pollLast(); + if (name == null) { + name = rootName; + rootName = null; + } + return name; + } + + @Override + public boolean isRoot() { + return segments.isEmpty(); + } + + @Override + public Collection getSegments() { + return segments; + } + + /** + * Returns the relative path using '{@literal /}' as the separator. + */ + @Override + public String toRelativePath() { + switch (segments.size()) { + case 0: + return ""; + case 1: + return segments.getLast(); + default: + int length = segments.size() - 1; + for (String segment : segments) { + length += segment.length(); + } + StringBuilder buffer = new StringBuilder(length); + Iterator iterator = segments.iterator(); + while (true) { + buffer.append(iterator.next()); + if (!iterator.hasNext()) { + break; + } + buffer.append('/'); + } + return buffer.toString(); + } + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/RelativePathTrackingFileSystemSnapshotHierarchyVisitor.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/RelativePathTrackingFileSystemSnapshotHierarchyVisitor.java new file mode 100644 index 00000000..16a6f2ac --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/RelativePathTrackingFileSystemSnapshotHierarchyVisitor.java @@ -0,0 +1,38 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import org.gradle.internal.RelativePathSupplier; + +public interface RelativePathTrackingFileSystemSnapshotHierarchyVisitor { + /** + * Called before visiting the contents of a directory. + */ + default void enterDirectory(DirectorySnapshot directorySnapshot, RelativePathSupplier relativePath) {} + + /** + * Called for each regular file/directory/missing/unavailable file. + * + * @return how to continue visiting the rest of the snapshot hierarchy. + */ + SnapshotVisitResult visitEntry(FileSystemLocationSnapshot snapshot, RelativePathSupplier relativePath); + + /** + * Called after all entries in the directory has been visited. + */ + default void leaveDirectory(DirectorySnapshot directorySnapshot, RelativePathSupplier relativePath) {} +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/RootTrackingFileSystemSnapshotHierarchyVisitor.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/RootTrackingFileSystemSnapshotHierarchyVisitor.java new file mode 100644 index 00000000..170ed1cb --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/RootTrackingFileSystemSnapshotHierarchyVisitor.java @@ -0,0 +1,55 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +public abstract class RootTrackingFileSystemSnapshotHierarchyVisitor implements FileSystemSnapshotHierarchyVisitor { + private int treeDepth; + + /** + * Called before visiting the contents of a directory. + */ + public void enterDirectory(DirectorySnapshot directorySnapshot, boolean isRoot) {} + + /** + * Called for each regular file/directory/missing/unavailable file. + * + * @return how to continue visiting the rest of the snapshot hierarchy. + */ + public abstract SnapshotVisitResult visitEntry(FileSystemLocationSnapshot snapshot, boolean isRoot); + + /** + * Called after all entries in the directory has been visited. + */ + public void leaveDirectory(DirectorySnapshot directorySnapshot, boolean isRoot) {} + + @Override + public final void enterDirectory(DirectorySnapshot directorySnapshot) { + enterDirectory(directorySnapshot, treeDepth == 0); + treeDepth++; + } + + @Override + public SnapshotVisitResult visitEntry(FileSystemLocationSnapshot snapshot) { + return visitEntry(snapshot, treeDepth == 0); + } + + @Override + public final void leaveDirectory(DirectorySnapshot directorySnapshot) { + treeDepth--; + leaveDirectory(directorySnapshot, treeDepth == 0); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SearchUtil.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SearchUtil.java new file mode 100644 index 00000000..26953b3c --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SearchUtil.java @@ -0,0 +1,72 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import java.util.Comparator; +import java.util.List; + +public abstract class SearchUtil { + + /** + * Does a binary search for an element determined by a {@link Comparable}. + * + * See {@link java.util.Collections#binarySearch(List, Object, Comparator)}. + * @param sortedElements {@link java.util.RandomAccess} list, sorted compatible with the comparable. + * @param key determines which element to search for. + * @return the index of the search key, if it is contained in the list; + * otherwise, (-(insertion point) - 1). The + * insertion point is defined as the point at which the + * key would be inserted into the list: the index of the first + * element greater than the key, or {@code list.size()} if all + * elements in the list are less than the specified key. Note + * that this guarantees that the return value will be >= 0 if + * and only if the key is found. + */ + public static int binarySearch(List sortedElements, Comparable key) { + int size = sortedElements.size(); + switch (size) { + case 0: + return -1; + case 1: + T onlyElement = sortedElements.get(0); + int comparedToSearch = key.compareTo(onlyElement); + return comparedToSearch == 0 + ? 0 + : comparedToSearch < 0 + ? -1 + : -2; + default: + int low = 0; + int high = size - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; + T midVal = sortedElements.get(mid); + int cmp = key.compareTo(midVal); + + if (cmp > 0) { + low = mid + 1; + } else if (cmp < 0) { + high = mid - 1; + } else { + return mid; // key found + } + } + return -(low + 1); // key not found + } + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SingletonChildMap.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SingletonChildMap.java new file mode 100644 index 00000000..fbf8569e --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SingletonChildMap.java @@ -0,0 +1,148 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import java.util.Collections; +import java.util.List; +import java.util.function.BiConsumer; + +import static org.gradle.internal.snapshot.ChildMapFactory.childMap; + +public class SingletonChildMap implements ChildMap { + private final Entry entry; + + public SingletonChildMap(String path, T child) { + this(new Entry<>(path, child)); + } + + public SingletonChildMap(Entry entry) { + this.entry = entry; + } + + @Override + public boolean isEmpty() { + return false; + } + + @Override + public List values() { + return Collections.singletonList(entry.getValue()); + } + + @Override + public List> entries() { + return Collections.singletonList(entry); + } + + @Override + public R withNode(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, NodeHandler handler) { + return entry.withNode(targetPath, caseSensitivity, handler); + } + + @Override + public ChildMap invalidate(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, InvalidationHandler handler) { + return entry.withNode(targetPath, caseSensitivity, new AbstractInvalidateChildHandler(handler) { + @SuppressWarnings("unchecked") + @Override + public SingletonChildMap getChildMap() { + return (SingletonChildMap) SingletonChildMap.this; + } + + @Override + public ChildMap withReplacedChild(RESULT newChild) { + return withReplacedChild(entry.getPath(), newChild); + } + + @Override + public ChildMap withReplacedChild(String newChildPath, RESULT newChild) { + return getChildMap().withReplacedChild(newChildPath, newChild); + } + + @Override + public ChildMap withRemovedChild() { + return EmptyChildMap.getInstance(); + } + }); + } + + @Override + public ChildMap store(VfsRelativePath targetPath, CaseSensitivity caseSensitivity, StoreHandler storeHandler) { + return entry.handlePath(targetPath, caseSensitivity, new AbstractStorePathRelationshipHandler(caseSensitivity, storeHandler) { + @Override + public ChildMap withReplacedChild(T newChild) { + return withReplacedChild(entry.getPath(), newChild); + } + + @Override + public ChildMap withReplacedChild(String newChildPath, T newChild) { + return SingletonChildMap.this.withReplacedChild(newChildPath, newChild); + } + + @Override + public ChildMap withNewChild(String newChildPath, T newChild) { + return SingletonChildMap.this.withNewChild(caseSensitivity, newChildPath, newChild); + } + + }); + } + + private ChildMap withNewChild(CaseSensitivity caseSensitivity, String newChildPath, T newChild) { + Entry newEntry = new Entry<>(newChildPath, newChild); + return childMap(caseSensitivity, entry, newEntry); + } + + private ChildMap withReplacedChild(String newPath, RESULT newChild) { + if (entry.getPath().equals(newPath) && entry.getValue().equals(newChild)) { + return castThis(); + } + return new SingletonChildMap<>(newPath, newChild); + } + + @SuppressWarnings("unchecked") + private SingletonChildMap castThis() { + return (SingletonChildMap) this; + } + + @Override + public void visitChildren(BiConsumer visitor) { + visitor.accept(entry.getPath(), entry.getValue()); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + SingletonChildMap that = (SingletonChildMap) o; + + return entry.equals(that.entry); + } + + @Override + public int hashCode() { + return entry.hashCode(); + } + + @Override + public String toString() { + return entry.toString(); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SnapshotHierarchy.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SnapshotHierarchy.java new file mode 100644 index 00000000..60b71bc2 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SnapshotHierarchy.java @@ -0,0 +1,120 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import javax.annotation.CheckReturnValue; +import java.util.Collection; +import java.util.Optional; + +/** + * An immutable hierarchy of snapshots of the file system. + * + * Intended to store an in-memory representation of the state of the file system. + */ +public interface SnapshotHierarchy { + + /** + * Returns the snapshot stored at the absolute path. + */ + Optional getMetadata(String absolutePath); + + /** + * Returns the snapshot stored at the absolute path. + */ + default Optional getSnapshot(String absolutePath) { + return getMetadata(absolutePath) + .filter(FileSystemLocationSnapshot.class::isInstance) + .map(FileSystemLocationSnapshot.class::cast); + } + + boolean hasDescendantsUnder(String absolutePath); + + /** + * Returns a hierarchy augmented by the information of the snapshot at the absolute path. + */ + @CheckReturnValue + SnapshotHierarchy store(String absolutePath, MetadataSnapshot snapshot, NodeDiffListener diffListener); + + /** + * Returns a hierarchy without any information at the absolute path. + */ + @CheckReturnValue + SnapshotHierarchy invalidate(String absolutePath, NodeDiffListener diffListener); + + /** + * The empty hierarchy. + */ + @CheckReturnValue + SnapshotHierarchy empty(); + + void visitSnapshotRoots(SnapshotVisitor snapshotVisitor); + + void visitSnapshotRoots(String absolutePath, SnapshotVisitor snapshotVisitor); + + interface SnapshotVisitor { + void visitSnapshotRoot(FileSystemLocationSnapshot snapshot); + } + + /** + * Receives diff when a {@link SnapshotHierarchy} is updated. + * + * Only the root nodes which have been removed/added are reported. + */ + interface NodeDiffListener { + NodeDiffListener NOOP = new NodeDiffListener() { + @Override + public void nodeRemoved(FileSystemNode node) { + } + + @Override + public void nodeAdded(FileSystemNode node) { + } + }; + + /** + * Called when a node is removed during the update. + * + * Only called for the node which is removed, and not every node in the hierarchy which is removed. + */ + void nodeRemoved(FileSystemNode node); + + /** + * Called when a node is added during the update. + * + * Only called for the node which is added, and not every node in the hierarchy which is added. + */ + void nodeAdded(FileSystemNode node); + } + + /** + * Listens to diffs to {@link FileSystemLocationSnapshot}s during an update of {@link SnapshotHierarchy}. + * + * Similar to {@link NodeDiffListener}, only that + * - it listens for {@link FileSystemLocationSnapshot}s and not {@link FileSystemNode}s. + * - it receives all the changes for one update at once. + */ + interface SnapshotDiffListener { + SnapshotDiffListener NOOP = (removedSnapshots, addedSnapshots) -> {}; + + /** + * Called after the update to {@link SnapshotHierarchy} finished. + * + * Only the roots of added/removed hierarchies are reported. + */ + void changed(Collection removedSnapshots, Collection addedSnapshots); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SnapshotUtil.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SnapshotUtil.java new file mode 100644 index 00000000..dc00e5e5 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SnapshotUtil.java @@ -0,0 +1,108 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.ImmutableMultimap; +import org.gradle.internal.hash.HashCode; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; + +public class SnapshotUtil { + + public static Map index(FileSystemSnapshot snapshot) { + HashMap index = new HashMap<>(); + snapshot.accept(entrySnapshot -> { + index.put(entrySnapshot.getAbsolutePath(), entrySnapshot); + return SnapshotVisitResult.CONTINUE; + }); + return index; + } + + public static Map rootIndex(FileSystemSnapshot snapshot) { + HashMap index = new HashMap<>(); + snapshot.accept(entrySnapshot -> { + index.put(entrySnapshot.getAbsolutePath(), entrySnapshot); + return SnapshotVisitResult.SKIP_SUBTREE; + }); + return index; + } + + public static Optional getMetadataFromChildren(ChildMap children, VfsRelativePath targetPath, CaseSensitivity caseSensitivity, Supplier> noChildFoundResult) { + return children.withNode(targetPath, caseSensitivity, new ChildMap.NodeHandler>() { + @Override + public Optional handleAsDescendantOfChild(VfsRelativePath pathInChild, T child) { + return child.getSnapshot(pathInChild, caseSensitivity); + } + + @Override + public Optional handleAsAncestorOfChild(String childPath, T child) { + return noChildFoundResult.get(); + } + + @Override + public Optional handleExactMatchWithChild(T child) { + return child.getSnapshot(); + } + + @Override + public Optional handleUnrelatedToAnyChild() { + return noChildFoundResult.get(); + } + }); + } + + public static ReadOnlyFileSystemNode getChild(ChildMap children, VfsRelativePath targetPath, CaseSensitivity caseSensitivity) { + return children.withNode(targetPath, caseSensitivity, new ChildMap.NodeHandler() { + @Override + public ReadOnlyFileSystemNode handleAsDescendantOfChild(VfsRelativePath pathInChild, T child) { + return child.getNode(pathInChild, caseSensitivity); + } + + @Override + public ReadOnlyFileSystemNode handleAsAncestorOfChild(String childPath, T child) { + // TODO: This is not correct, it should be a node with the child at targetPath.fromChild(childPath). + return child; + } + + @Override + public ReadOnlyFileSystemNode handleExactMatchWithChild(T child) { + return child; + } + + @Override + public ReadOnlyFileSystemNode handleUnrelatedToAnyChild() { + return ReadOnlyFileSystemNode.EMPTY; + } + }); + } + + public static ImmutableMultimap getRootHashes(FileSystemSnapshot roots) { + if (roots == FileSystemSnapshot.EMPTY) { + return ImmutableMultimap.of(); + } + ImmutableMultimap.Builder builder = ImmutableListMultimap.builder(); + roots.accept(snapshot -> { + builder.put(snapshot.getAbsolutePath(), snapshot.getHash()); + return SnapshotVisitResult.SKIP_SUBTREE; + }); + return builder.build(); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SnapshotVisitResult.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SnapshotVisitResult.java new file mode 100644 index 00000000..56b713b9 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SnapshotVisitResult.java @@ -0,0 +1,42 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +/** + * Ways to continue visiting a snapshot hierarchy after an entry has been visited. + * + * @see java.nio.file.FileVisitResult + */ +public enum SnapshotVisitResult { + + /** + * Continue visiting. When returned after visiting a directory, + * the entries in the directory will be visited next. + */ + CONTINUE, + + /** + * Terminate visiting immediately. + */ + TERMINATE, + + /** + * If returned from visiting a directory, the directories entries will not be visited; + * otherwise works as {@link #CONTINUE}. + */ + SKIP_SUBTREE +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SnapshottingFilter.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SnapshottingFilter.java new file mode 100644 index 00000000..ab5f151e --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/SnapshottingFilter.java @@ -0,0 +1,50 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import java.nio.file.Path; + +public interface SnapshottingFilter { + SnapshottingFilter EMPTY = new SnapshottingFilter() { + @Override + public boolean isEmpty() { + return true; + } + + @Override + public FileSystemSnapshotPredicate getAsSnapshotPredicate() { + return (location, relativePath) -> true; + } + + @Override + public DirectoryWalkerPredicate getAsDirectoryWalkerPredicate() { + return (path, name, isDirectory, relativePath) -> true; + } + }; + + boolean isEmpty(); + FileSystemSnapshotPredicate getAsSnapshotPredicate(); + DirectoryWalkerPredicate getAsDirectoryWalkerPredicate(); + + interface DirectoryWalkerPredicate { + boolean test(Path path, String name, boolean isDirectory, Iterable relativePath); + } + + interface FileSystemSnapshotPredicate { + boolean test(FileSystemLocationSnapshot fileSystemLocation, Iterable relativePath); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/UnknownFileSystemNode.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/UnknownFileSystemNode.java new file mode 100644 index 00000000..a05dea84 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/UnknownFileSystemNode.java @@ -0,0 +1,53 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import java.util.Optional; + +/** + * An incomplete snapshot where we don’t know if it’s a file, a directory, or nothing. + * + * The snapshot must have children. + * It is created when we store missing files underneath it, so that we don’t have to query them again and again. + */ +public class UnknownFileSystemNode extends AbstractIncompleteFileSystemNode { + + public UnknownFileSystemNode(ChildMap children) { + super(children); + assert !children.isEmpty(); + } + + @Override + public Optional getSnapshot() { + return Optional.empty(); + } + + @Override + protected FileSystemNode withIncompleteChildren(ChildMap merged) { + return new UnknownFileSystemNode(merged); + } + + @Override + protected Optional withAllChildrenRemoved() { + return Optional.empty(); + } + + @Override + protected FileSystemNode withIncompleteChildren() { + return this; + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ValueSnapshot.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ValueSnapshot.java new file mode 100644 index 00000000..29391f90 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ValueSnapshot.java @@ -0,0 +1,34 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import org.gradle.internal.hash.Hashable; + +import javax.annotation.Nullable; + +/** + * An immutable snapshot of the state of some Java object or object graph. + * + *

Implementations are not required to be able to recreate the object, and should retain as little state as possible. + * In particular, implementations should not hold on to user ClassLoaders.

+ */ +public interface ValueSnapshot extends Hashable { + /** + * Takes a snapshot of the given value, using this as a candidate snapshot. If the value is the same as the value represented by this snapshot, this snapshot must be returned. + */ + ValueSnapshot snapshot(@Nullable Object value, ValueSnapshotter snapshotter); +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ValueSnapshotter.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ValueSnapshotter.java new file mode 100644 index 00000000..8df258bc --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ValueSnapshotter.java @@ -0,0 +1,37 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import javax.annotation.Nullable; + +public interface ValueSnapshotter { + /** + * Creates a {@link ValueSnapshot} of the given value, that contains a snapshot of the current state of the value. A snapshot represents an immutable fingerprint of the value that can be later used to determine if a value has changed. + * + *

The snapshots must contain no references to the ClassLoader of the value.

+ * + * @throws ValueSnapshottingException On failure to snapshot the value. + */ + ValueSnapshot snapshot(@Nullable Object value) throws ValueSnapshottingException; + + /** + * Creates a snapshot of the given value, given a candidate snapshot. If the value is the same as the value provided by the candidate snapshot, the candidate must be returned. + * + * @throws ValueSnapshottingException On failure to snapshot the value. + */ + ValueSnapshot snapshot(@Nullable Object value, ValueSnapshot candidate) throws ValueSnapshottingException; +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ValueSnapshottingException.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ValueSnapshottingException.java new file mode 100644 index 00000000..3928dfc1 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/ValueSnapshottingException.java @@ -0,0 +1,29 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import javax.annotation.Nullable; + +public class ValueSnapshottingException extends RuntimeException { + public ValueSnapshottingException(String message) { + super(message); + } + + public ValueSnapshottingException(String message, @Nullable Throwable cause) { + super(message, cause); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/VfsRelativePath.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/VfsRelativePath.java new file mode 100644 index 00000000..335ee55f --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/VfsRelativePath.java @@ -0,0 +1,286 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot; + +import com.google.common.annotations.VisibleForTesting; + +import static org.gradle.internal.snapshot.CaseSensitivity.CASE_SENSITIVE; +import static org.gradle.internal.snapshot.PathUtil.compareChars; +import static org.gradle.internal.snapshot.PathUtil.compareCharsIgnoringCase; +import static org.gradle.internal.snapshot.PathUtil.equalChars; +import static org.gradle.internal.snapshot.PathUtil.isFileSeparator; + +/** + * A relative path represented by a path suffix of an absolute path. + * + * The use of this class is to improve performance by avoiding to call {@link String#substring(int)}. + * The class represents the relative path of absolutePath.substring(offset). + * + * A relative path does not start or end with a slash. + */ +public class VfsRelativePath { + private final String absolutePath; + private final int offset; + + /** + * The relative path from the root of the file system for the given absolute path. + * + * E.g.: + * 'C:/' -> 'C:' + * '/home/user/project' -> 'home/user/project' + * '/' -> '' + * '//uncpath/relative/path' -> 'uncpath/relative/path' + * 'C:/Users/user/project' -> 'C:/Users/user/project' + */ + public static VfsRelativePath of(String absolutePath) { + String normalizedRoot = normalizeRoot(absolutePath); + return VfsRelativePath.of(normalizedRoot, determineOffset(normalizedRoot)); + } + + @VisibleForTesting + static VfsRelativePath of(String absolutePath, int offset) { + return new VfsRelativePath(absolutePath, offset); + } + + private static String normalizeRoot(String absolutePath) { + if (absolutePath.equals("/")) { + return absolutePath; + } + return isFileSeparator(absolutePath.charAt(absolutePath.length() - 1)) + ? absolutePath.substring(0, absolutePath.length() - 1) + : absolutePath; + } + + private static int determineOffset(String absolutePath) { + for (int i = 0; i < absolutePath.length(); i++) { + if (!isFileSeparator(absolutePath.charAt(i))) { + return i; + } + } + return absolutePath.length(); + } + + private VfsRelativePath(String absolutePath, int offset) { + this.absolutePath = absolutePath; + this.offset = offset; + } + + /** + * Returns a new relative path starting from the given start index. + * + * E.g. + * (some/path, 5) -> path + */ + public VfsRelativePath suffixStartingFrom(int startIndex) { + return new VfsRelativePath(absolutePath, offset + startIndex); + } + + /** + * Returns a new relative path starting from the child. + * + * E.g. + * (some/path, some) -> path + * (some/path/other, some) -> path/other + * (C:, '') -> C: + */ + public VfsRelativePath fromChild(String relativeChildPath) { + return relativeChildPath.isEmpty() + ? this + : suffixStartingFrom(relativeChildPath.length() + 1); + } + + public int length() { + return absolutePath.length() - offset; + } + + /** + * The relative path represented by this suffix as a String. + */ + public String getAsString() { + return absolutePath.substring(offset); + } + + public String getAbsolutePath() { + return absolutePath; + } + + /** + * Returns the length of the common prefix of this with a relative path. + * + * The length of the common prefix does not include the last line separator. + * + * Examples: + * lengthOfCommonPrefix("some/path", "some/other") == 4 + * lengthOfCommonPrefix("some/path", "some1/other") == 0 + * lengthOfCommonPrefix("some/longer/path", "some/longer/other") == 11 + * lengthOfCommonPrefix("some/longer", "some/longer/path") == 11 + */ + public int lengthOfCommonPrefix(String relativePath, CaseSensitivity caseSensitivity) { + int pos = 0; + int lastSeparator = 0; + int maxPos = Math.min(relativePath.length(), absolutePath.length() - offset); + for (; pos < maxPos; pos++) { + char charInPath1 = relativePath.charAt(pos); + char charInPath2 = absolutePath.charAt(pos + offset); + if (!equalChars(charInPath1, charInPath2, caseSensitivity)) { + break; + } + if (isFileSeparator(charInPath1)) { + lastSeparator = pos; + } + } + if (pos == maxPos) { + if (relativePath.length() == absolutePath.length() - offset) { + return pos; + } + if (pos < relativePath.length() && isFileSeparator(relativePath.charAt(pos))) { + return pos; + } + if (pos < absolutePath.length() - offset && isFileSeparator(absolutePath.charAt(pos + offset))) { + return pos; + } + } + return lastSeparator; + } + + /** + * Compares to the first segment of a relative path. + * + * A segment of a path is the part between two file separators. + * For example, the path some/long/path has the segments some, long and path. + * + * Similar to {@link #lengthOfCommonPrefix(String, CaseSensitivity)}, + * only that this method compares to the first segment of the path if there is no common prefix. + * + * The path must not start with a separator. + * + * For example, this method returns: + * some/path == some/other + * some1/path < some2/other + * some/path > some1/other + * some/same == some/same/more + * some/one/alma == some/two/bela + * a/some < b/other + * + * @return 0 if the two paths have a common prefix, and the comparison of the first segment of each path if not. + */ + public int compareToFirstSegment(String relativePath, CaseSensitivity caseSensitivity) { + int maxPos = Math.min(relativePath.length(), absolutePath.length() - offset); + int accumulatedValue = 0; + for (int pos = 0; pos < maxPos; pos++) { + char charInPath1 = absolutePath.charAt(pos + offset); + char charInPath2 = relativePath.charAt(pos); + int comparedChars = compareCharsIgnoringCase(charInPath1, charInPath2); + if (comparedChars != 0) { + return comparedChars; + } + accumulatedValue = computeCombinedCompare(accumulatedValue, charInPath1, charInPath2, caseSensitivity == CASE_SENSITIVE); + if (isFileSeparator(charInPath1)) { + if (pos > 0) { + return accumulatedValue; + } + } + } + if (absolutePath.length() - offset == relativePath.length()) { + return accumulatedValue; + } + if (absolutePath.length() - offset > relativePath.length()) { + return isFileSeparator(absolutePath.charAt(maxPos + offset)) ? accumulatedValue : 1; + } + return isFileSeparator(relativePath.charAt(maxPos)) ? accumulatedValue : -1; + } + + /** + * Checks whether this path has the prefix. + */ + public boolean hasPrefix(String prefix, CaseSensitivity caseSensitivity) { + int prefixLength = prefix.length(); + if (prefixLength == 0) { + return true; + } + int pathLength = absolutePath.length(); + int endOfThisSegment = prefixLength + offset; + if (pathLength < endOfThisSegment) { + return false; + } + for (int i = prefixLength - 1, j = endOfThisSegment - 1; i >= 0; i--, j--) { + if (!equalChars(prefix.charAt(i), absolutePath.charAt(j), caseSensitivity)) { + return false; + } + } + return endOfThisSegment == pathLength || isFileSeparator(absolutePath.charAt(endOfThisSegment)); + } + + /** + * Checks whether this path is a prefix of another path. + */ + public boolean isPrefixOf(String otherPath, CaseSensitivity caseSensitivity) { + int prefixLength = length(); + if (prefixLength == 0) { + return true; + } + int endOfThisSegment = prefixLength + offset; + int pathLength = otherPath.length(); + if (pathLength < prefixLength) { + return false; + } + for (int i = prefixLength - 1, j = endOfThisSegment - 1; i >= 0; i--, j--) { + if (!equalChars(otherPath.charAt(i), absolutePath.charAt(j), caseSensitivity)) { + return false; + } + } + return prefixLength == pathLength || isFileSeparator(otherPath.charAt(prefixLength)); + } + + private static int computeCombinedCompare(int previousCombinedValue, char charInPath1, char charInPath2, boolean caseSensitive) { + if (!caseSensitive) { + return 0; + } + return previousCombinedValue == 0 + ? compareChars(charInPath1, charInPath2) + : previousCombinedValue; + } + + @Override + public String toString() { + return getAsString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + VfsRelativePath that = (VfsRelativePath) o; + + if (offset != that.offset) { + return false; + } + return absolutePath.equals(that.absolutePath); + } + + @Override + public int hashCode() { + int result = absolutePath.hashCode(); + result = 31 * result + offset; + return result; + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/DirectorySnapshotter.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/DirectorySnapshotter.java new file mode 100644 index 00000000..2f96919d --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/DirectorySnapshotter.java @@ -0,0 +1,431 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot.impl; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Interner; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import org.gradle.internal.file.FileMetadata; +import org.gradle.internal.file.FileMetadata.AccessType; +import org.gradle.internal.file.impl.DefaultFileMetadata; +import org.gradle.internal.hash.FileHasher; +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.snapshot.DirectorySnapshot; +import org.gradle.internal.snapshot.FileSystemLeafSnapshot; +import org.gradle.internal.snapshot.FileSystemLocationSnapshot; +import org.gradle.internal.snapshot.MerkleDirectorySnapshotBuilder; +import org.gradle.internal.snapshot.MissingFileSnapshot; +import org.gradle.internal.snapshot.RegularFileSnapshot; +import org.gradle.internal.snapshot.RelativePathTracker; +import org.gradle.internal.snapshot.SnapshottingFilter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.CheckReturnValue; +import javax.annotation.Nullable; +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.FileSystemLoopException; +import java.nio.file.FileVisitOption; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.Deque; +import java.util.EnumSet; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Predicate; + +import static org.gradle.internal.snapshot.MerkleDirectorySnapshotBuilder.EmptyDirectoryHandlingStrategy.INCLUDE_EMPTY_DIRS; + +public class DirectorySnapshotter { + private static final Logger LOGGER = LoggerFactory.getLogger(DirectorySnapshotter.class); + private static final EnumSet DONT_FOLLOW_SYMLINKS = EnumSet.noneOf(FileVisitOption.class); + private static final SymbolicLinkMapping EMPTY_SYMBOLIC_LINK_MAPPING = new SymbolicLinkMapping() { + + @Override + public String remapAbsolutePath(Path path) { + return path.toString(); + } + + @Override + public SymbolicLinkMapping withNewMapping(String source, String target, RelativePathTracker currentPathTracker) { + return new DefaultSymbolicLinkMapping(source, target, currentPathTracker.getSegments()); + } + + @Override + public Iterable getRemappedSegments(Iterable segments) { + return segments; + } + }; + + private final FileHasher hasher; + private final Interner stringInterner; + private final DefaultExcludes defaultExcludes; + private final DirectorySnapshotterStatistics.Collector collector; + + public DirectorySnapshotter(FileHasher hasher, Interner stringInterner, Collection defaultExcludes, DirectorySnapshotterStatistics.Collector collector) { + this.hasher = hasher; + this.stringInterner = stringInterner; + this.defaultExcludes = new DefaultExcludes(defaultExcludes); + this.collector = collector; + } + + public FileSystemLocationSnapshot snapshot(String absolutePath, @Nullable SnapshottingFilter.DirectoryWalkerPredicate predicate, final AtomicBoolean hasBeenFiltered) { + try { + Path rootPath = Paths.get(absolutePath); + PathVisitor visitor = new PathVisitor(predicate, hasBeenFiltered, hasher, stringInterner, defaultExcludes, collector, EMPTY_SYMBOLIC_LINK_MAPPING); + Files.walkFileTree(rootPath, DONT_FOLLOW_SYMLINKS, Integer.MAX_VALUE, visitor); + return visitor.getResult(); + } catch (IOException e) { + throw new UncheckedIOException(String.format("Could not list contents of directory '%s'.", absolutePath), e); + } + } + + private interface SymbolicLinkMapping { + String remapAbsolutePath(Path path); + @CheckReturnValue + SymbolicLinkMapping withNewMapping(String source, String target, RelativePathTracker currentPathTracker); + Iterable getRemappedSegments(Iterable segments); + } + + private static class DefaultSymbolicLinkMapping implements SymbolicLinkMapping { + private final String sourcePath; + private final String targetPath; + private final Iterable prefixRelativePath; + + public DefaultSymbolicLinkMapping(String sourcePath, String targetPath, Iterable prefixRelativePath) { + this.sourcePath = sourcePath; + this.targetPath = targetPath; + this.prefixRelativePath = prefixRelativePath; + } + + @Override + public String remapAbsolutePath(Path dir) { + return remapAbsolutePath(dir.toString()); + } + + public String remapAbsolutePath(String absolutePath) { + if (absolutePath.equals(targetPath)) { + return sourcePath; + } + if (absolutePath.startsWith(targetPath) && absolutePath.charAt(targetPath.length()) == File.separatorChar) { + return sourcePath + File.separatorChar + absolutePath.substring(targetPath.length() + 1); + } + throw new IllegalArgumentException("Cannot remap path '" + absolutePath + "' which does not have '" + targetPath + "' as a prefix"); + } + + @Override + public SymbolicLinkMapping withNewMapping(String source, String target, RelativePathTracker currentPathTracker) { + return new DefaultSymbolicLinkMapping(remapAbsolutePath(source), target, getRemappedSegments(currentPathTracker.getSegments())); + } + + @Override + public Iterable getRemappedSegments(Iterable segments) { + return Iterables.concat(prefixRelativePath, segments); + } + } + + @VisibleForTesting + static class DefaultExcludes { + private final ImmutableSet excludeFileNames; + private final ImmutableSet excludedDirNames; + private final Predicate excludedFileNameSpec; + + public DefaultExcludes(Collection defaultExcludes) { + final List excludeFiles = Lists.newArrayList(); + final List excludeDirs = Lists.newArrayList(); + final List> excludeFileSpecs = Lists.newArrayList(); + for (String defaultExclude : defaultExcludes) { + if (defaultExclude.startsWith("**/")) { + defaultExclude = defaultExclude.substring(3); + } + int length = defaultExclude.length(); + if (defaultExclude.endsWith("/**")) { + excludeDirs.add(defaultExclude.substring(0, length - 3)); + } else { + int firstStar = defaultExclude.indexOf('*'); + if (firstStar == -1) { + excludeFiles.add(defaultExclude); + } else { + Predicate start = firstStar == 0 + ? it -> true + : new StartMatcher(defaultExclude.substring(0, firstStar)); + Predicate end = firstStar == length - 1 + ? it -> true + : new EndMatcher(defaultExclude.substring(firstStar + 1, length)); + excludeFileSpecs.add(start.and(end)); + } + } + } + + this.excludeFileNames = ImmutableSet.copyOf(excludeFiles); + this.excludedFileNameSpec = excludeFileSpecs.stream().reduce(it -> false, Predicate::or); + this.excludedDirNames = ImmutableSet.copyOf(excludeDirs); + } + + public boolean excludeDir(String name) { + return excludedDirNames.contains(name); + } + + public boolean excludeFile(String name) { + return excludeFileNames.contains(name) || excludedFileNameSpec.test(name); + } + + private static class EndMatcher implements Predicate { + private final String end; + + public EndMatcher(String end) { + this.end = end; + } + + @Override + public boolean test(String element) { + return element.endsWith(end); + } + } + + private static class StartMatcher implements Predicate { + private final String start; + + public StartMatcher(String start) { + this.start = start; + } + + @Override + public boolean test(String element) { + return element.startsWith(start); + } + } + } + + private static class PathVisitor extends DirectorySnapshotterStatistics.CollectingFileVisitor { + private final RelativePathTracker pathTracker = new RelativePathTracker(); + private final MerkleDirectorySnapshotBuilder builder; + private final SnapshottingFilter.DirectoryWalkerPredicate predicate; + private final AtomicBoolean hasBeenFiltered; + private final FileHasher hasher; + private final Interner stringInterner; + private final DefaultExcludes defaultExcludes; + private final SymbolicLinkMapping symbolicLinkMapping; + private final Deque parentDirectories = new ArrayDeque<>(); + + public PathVisitor( + @Nullable SnapshottingFilter.DirectoryWalkerPredicate predicate, + AtomicBoolean hasBeenFiltered, + FileHasher hasher, + Interner stringInterner, + DefaultExcludes defaultExcludes, + DirectorySnapshotterStatistics.Collector statisticsCollector, + SymbolicLinkMapping symbolicLinkMapping + ) { + super(statisticsCollector); + this.builder = MerkleDirectorySnapshotBuilder.sortingRequired(); + this.predicate = predicate; + this.hasBeenFiltered = hasBeenFiltered; + this.hasher = hasher; + this.stringInterner = stringInterner; + this.defaultExcludes = defaultExcludes; + this.symbolicLinkMapping = symbolicLinkMapping; + } + + @Override + protected FileVisitResult doPreVisitDirectory(Path dir, BasicFileAttributes attrs) { + String fileName = getInternedFileName(dir); + pathTracker.enter(fileName); + if (pathTracker.isRoot() || shouldVisit(dir, fileName, true, pathTracker.getSegments())) { + builder.enterDirectory(AccessType.DIRECT, intern(symbolicLinkMapping.remapAbsolutePath(dir)), fileName, INCLUDE_EMPTY_DIRS); + parentDirectories.addFirst(dir.toString()); + return FileVisitResult.CONTINUE; + } else { + pathTracker.leave(); + return FileVisitResult.SKIP_SUBTREE; + } + } + + @Override + protected FileVisitResult doVisitFile(Path file, BasicFileAttributes attrs) { + String internedFileName = getInternedFileName(file); + pathTracker.enter(internedFileName); + try { + if (attrs.isSymbolicLink()) { + BasicFileAttributes targetAttributes = readAttributesOfSymlinkTarget(file, attrs); + if (targetAttributes.isDirectory()) { + try { + Path targetDir = file.toRealPath(); + String targetDirString = targetDir.toString(); + if (introducesCycle(targetDirString)) { + return FileVisitResult.CONTINUE; + } + if (pathTracker.isRoot() || shouldVisit(targetDir, internedFileName, true, pathTracker.getSegments())) { + PathVisitor subtreeVisitor = new PathVisitor( + predicate, + hasBeenFiltered, + hasher, + stringInterner, + defaultExcludes, + collector, + symbolicLinkMapping.withNewMapping(file.toString(), targetDirString, pathTracker) + ); + Files.walkFileTree(targetDir, EnumSet.noneOf(FileVisitOption.class), Integer.MAX_VALUE, subtreeVisitor); + DirectorySnapshot result = (DirectorySnapshot) subtreeVisitor.getResult(); + builder.visitDirectory(new DirectorySnapshot( + result.getAbsolutePath(), + internedFileName, + AccessType.VIA_SYMLINK, + result.getHash(), + result.getChildren() + )); + } + } catch (IOException e) { + throw new UncheckedIOException(String.format("Could not list contents of directory '%s'.", file), e); + } + } else { + visitResolvedFile(file, targetAttributes, AccessType.VIA_SYMLINK); + } + } else { + visitResolvedFile(file, attrs, AccessType.DIRECT); + } + return FileVisitResult.CONTINUE; + } finally { + pathTracker.leave(); + } + } + + private boolean introducesCycle(String targetDirString) { + return parentDirectories.contains(targetDirString); + } + + private void visitResolvedFile(Path file, BasicFileAttributes targetAttributes, AccessType accessType) { + String internedName = intern(file.getFileName().toString()); + if (shouldVisit(file, internedName, false, pathTracker.getSegments())) { + builder.visitLeafElement(snapshotFile(file, internedName, targetAttributes, accessType)); + } + } + + private BasicFileAttributes readAttributesOfSymlinkTarget(Path symlink, BasicFileAttributes symlinkAttributes) { + try { + return Files.readAttributes(symlink, BasicFileAttributes.class); + } catch (IOException ioe) { + // We emulate the behavior of `Files.walkFileTree(Path, EnumSet.of(FileVisitOption.FOLLOW_LINKS), PathVisitor)`, + // and return the attributes of the symlink if we can't read the attributes of the target of the symlink. + return symlinkAttributes; + } + } + + private FileSystemLeafSnapshot snapshotFile(Path absoluteFilePath, String internedName, BasicFileAttributes attrs, AccessType accessType) { + String internedRemappedAbsoluteFilePath = intern(symbolicLinkMapping.remapAbsolutePath(absoluteFilePath)); + if (attrs.isRegularFile()) { + try { + long lastModified = attrs.lastModifiedTime().toMillis(); + long fileLength = attrs.size(); + FileMetadata metadata = DefaultFileMetadata.file(lastModified, fileLength, accessType); + HashCode hash = hasher.hash(absoluteFilePath.toFile(), fileLength, lastModified); + return new RegularFileSnapshot(internedRemappedAbsoluteFilePath, internedName, hash, metadata); + } catch (UncheckedIOException e) { + LOGGER.info("Could not read file path '{}'.", absoluteFilePath, e); + } + } + return new MissingFileSnapshot(internedRemappedAbsoluteFilePath, internedName, accessType); + } + + /** unlistable directories (and maybe some locked files) will stop here */ + @Override + protected FileVisitResult doVisitFileFailed(Path file, IOException exc) { + String internedFileName = getInternedFileName(file); + pathTracker.enter(internedFileName); + try { + // File loop exceptions are ignored. When we encounter a loop (via symbolic links), we continue + // so we include all the other files apart from the loop. + // This way, we include each file only once. + if (isNotFileSystemLoopException(exc)) { + boolean isDirectory = Files.isDirectory(file); + if (shouldVisit(file, internedFileName, isDirectory, pathTracker.getSegments())) { + LOGGER.info("Could not read file path '{}'.", file); + String internedAbsolutePath = intern(file.toString()); + builder.visitLeafElement(new MissingFileSnapshot(internedAbsolutePath, internedFileName, AccessType.DIRECT)); + } + } + return FileVisitResult.CONTINUE; + } finally { + pathTracker.leave(); + } + } + + @Override + protected FileVisitResult doPostVisitDirectory(Path dir, IOException exc) { + pathTracker.leave(); + // File loop exceptions are ignored. When we encounter a loop (via symbolic links), we continue + // so we include all the other files apart from the loop. + // This way, we include each file only once. + if (isNotFileSystemLoopException(exc)) { + throw new UncheckedIOException(String.format("Could not read directory path '%s'.", dir), exc); + } + builder.leaveDirectory(); + parentDirectories.removeFirst(); + return FileVisitResult.CONTINUE; + } + + private boolean isNotFileSystemLoopException(@Nullable IOException e) { + return e != null && !(e instanceof FileSystemLoopException); + } + + private String intern(String string) { + return stringInterner.intern(string); + } + + /** + * Returns whether we want to visit the given path during our walk, or ignore it completely, + * based on the directory/file excludes or the provided filtering predicate. + * Excludes won't mark this walk as `filtered`, only if the `predicate` rejects any entry. + **/ + private boolean shouldVisit(Path path, String internedName, boolean isDirectory, Iterable relativePath) { + if (isDirectory) { + if (defaultExcludes.excludeDir(internedName)) { + return false; + } + } else if (defaultExcludes.excludeFile(internedName)) { + return false; + } + + if (predicate == null) { + return true; + } + boolean allowed = predicate.test(path, internedName, isDirectory, symbolicLinkMapping.getRemappedSegments(relativePath)); + if (!allowed) { + hasBeenFiltered.set(true); + } + return allowed; + } + + private String getInternedFileName(Path dir) { + Path fileName = dir.getFileName(); + return fileName == null ? "" : intern(fileName.toString()); + } + + public FileSystemLocationSnapshot getResult() { + return builder.getResult(); + } + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/DirectorySnapshotterStatistics.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/DirectorySnapshotterStatistics.java new file mode 100644 index 00000000..ea4582e7 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/DirectorySnapshotterStatistics.java @@ -0,0 +1,145 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot.impl; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.MessageFormat; +import java.util.concurrent.atomic.AtomicLong; + +public interface DirectorySnapshotterStatistics { + /** + * The number of visited directory trees. + */ + long getVisitedHierarchyCount(); + + /** + * The number of visited directories. + */ + long getVisitedDirectoryCount(); + + /** + * The number of visited files. + */ + long getVisitedFiles(); + + /** + * The number of files we failed to visit. + */ + long getFailedFiles(); + + class Collector { + private final AtomicLong hierarchyCount = new AtomicLong(); + private final AtomicLong directoryCount = new AtomicLong(); + private final AtomicLong fileCount = new AtomicLong(); + private final AtomicLong failedFileCount = new AtomicLong(); + + public void recordVisitHierarchy() { + hierarchyCount.incrementAndGet(); + } + + public void recordVisitDirectory() { + directoryCount.incrementAndGet(); + } + + public void recordVisitFile() { + fileCount.incrementAndGet(); + } + + public void recordVisitFileFailed() { + failedFileCount.incrementAndGet(); + } + + public DirectorySnapshotterStatistics collect() { + long hierarchyCount = this.hierarchyCount.getAndSet(0); + long directoryCount = this.directoryCount.getAndSet(0); + long fileCount = this.fileCount.getAndSet(0); + long failedFileCount = this.failedFileCount.getAndSet(0); + + return new DirectorySnapshotterStatistics() { + @Override + public long getVisitedHierarchyCount() { + return hierarchyCount; + } + + @Override + public long getVisitedDirectoryCount() { + return directoryCount; + } + + @Override + public long getVisitedFiles() { + return fileCount; + } + + @Override + public long getFailedFiles() { + return failedFileCount; + } + + @Override + public String toString() { + return MessageFormat.format("Snapshot {0,number,integer} directory hierarchies (visited {1,number,integer} directories, {2,number,integer} files and {3,number,integer} failed files)", + hierarchyCount, directoryCount, fileCount, failedFileCount); + } + }; + } + } + + abstract class CollectingFileVisitor implements FileVisitor { + protected final Collector collector; + + public CollectingFileVisitor(Collector collector) { + this.collector = collector; + collector.recordVisitHierarchy(); + } + + @Override + public final FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) { + collector.recordVisitDirectory(); + return doPreVisitDirectory(dir, attrs); + } + + protected abstract FileVisitResult doPreVisitDirectory(Path dir, BasicFileAttributes attrs); + + @Override + public final FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + collector.recordVisitFile(); + return doVisitFile(file, attrs); + } + + protected abstract FileVisitResult doVisitFile(Path file, BasicFileAttributes attrs); + + @Override + public final FileVisitResult visitFileFailed(Path file, IOException exc) { + collector.recordVisitFileFailed(); + return doVisitFileFailed(file, exc); + } + + protected abstract FileVisitResult doVisitFileFailed(Path file, IOException exc); + + @Override + public final FileVisitResult postVisitDirectory(Path dir, IOException exc) { + return doPostVisitDirectory(dir, exc); + } + + protected abstract FileVisitResult doPostVisitDirectory(Path dir, IOException exc); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/FileSystemSnapshotFilter.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/FileSystemSnapshotFilter.java new file mode 100644 index 00000000..27b1255b --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/FileSystemSnapshotFilter.java @@ -0,0 +1,109 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot.impl; + +import com.google.common.collect.ImmutableList; +import org.gradle.internal.RelativePathSupplier; +import org.gradle.internal.snapshot.DirectorySnapshot; +import org.gradle.internal.snapshot.FileSystemLeafSnapshot; +import org.gradle.internal.snapshot.FileSystemLocationSnapshot; +import org.gradle.internal.snapshot.FileSystemLocationSnapshot.FileSystemLocationSnapshotTransformer; +import org.gradle.internal.snapshot.FileSystemSnapshot; +import org.gradle.internal.snapshot.MerkleDirectorySnapshotBuilder; +import org.gradle.internal.snapshot.MissingFileSnapshot; +import org.gradle.internal.snapshot.RegularFileSnapshot; +import org.gradle.internal.snapshot.RelativePathTracker; +import org.gradle.internal.snapshot.RelativePathTrackingFileSystemSnapshotHierarchyVisitor; +import org.gradle.internal.snapshot.SnapshotVisitResult; +import org.gradle.internal.snapshot.SnapshottingFilter; + +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.gradle.internal.snapshot.MerkleDirectorySnapshotBuilder.EmptyDirectoryHandlingStrategy.INCLUDE_EMPTY_DIRS; + +public class FileSystemSnapshotFilter { + + private FileSystemSnapshotFilter() { + } + + public static FileSystemSnapshot filterSnapshot(SnapshottingFilter.FileSystemSnapshotPredicate predicate, FileSystemSnapshot unfiltered) { + MerkleDirectorySnapshotBuilder builder = MerkleDirectorySnapshotBuilder.noSortingRequired(); + AtomicBoolean hasBeenFiltered = new AtomicBoolean(false); + unfiltered.accept(new RelativePathTracker(), new FilteringVisitor(predicate, builder, hasBeenFiltered)); + if (builder.getResult() == null) { + return FileSystemSnapshot.EMPTY; + } + return hasBeenFiltered.get() ? builder.getResult() : unfiltered; + } + + private static class FilteringVisitor implements RelativePathTrackingFileSystemSnapshotHierarchyVisitor { + private final SnapshottingFilter.FileSystemSnapshotPredicate predicate; + private final MerkleDirectorySnapshotBuilder builder; + private final AtomicBoolean hasBeenFiltered; + + public FilteringVisitor(SnapshottingFilter.FileSystemSnapshotPredicate predicate, MerkleDirectorySnapshotBuilder builder, AtomicBoolean hasBeenFiltered) { + this.predicate = predicate; + this.builder = builder; + this.hasBeenFiltered = hasBeenFiltered; + } + + @Override + public void enterDirectory(DirectorySnapshot directorySnapshot, RelativePathSupplier relativePath) { + builder.enterDirectory(directorySnapshot, INCLUDE_EMPTY_DIRS); + } + + @Override + public SnapshotVisitResult visitEntry(FileSystemLocationSnapshot snapshot, RelativePathSupplier relativePath) { + boolean root = relativePath.isRoot(); + Iterable relativePathForFiltering = root + ? ImmutableList.of(snapshot.getName()) + : relativePath.getSegments(); + SnapshotVisitResult result; + boolean forceInclude = snapshot.accept(new FileSystemLocationSnapshotTransformer() { + @Override + public Boolean visitDirectory(DirectorySnapshot directorySnapshot) { + return root; + } + + @Override + public Boolean visitRegularFile(RegularFileSnapshot fileSnapshot) { + return false; + } + + @Override + public Boolean visitMissing(MissingFileSnapshot missingSnapshot) { + return false; + } + }); + if (forceInclude || predicate.test(snapshot, relativePathForFiltering)) { + if (snapshot instanceof FileSystemLeafSnapshot) { + builder.visitLeafElement((FileSystemLeafSnapshot) snapshot); + } + result = SnapshotVisitResult.CONTINUE; + } else { + hasBeenFiltered.set(true); + result = SnapshotVisitResult.SKIP_SUBTREE; + } + return result; + } + + @Override + public void leaveDirectory(DirectorySnapshot directorySnapshot, RelativePathSupplier relativePath) { + builder.leaveDirectory(); + } + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/ImplementationSnapshot.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/ImplementationSnapshot.java new file mode 100644 index 00000000..2ae2c297 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/ImplementationSnapshot.java @@ -0,0 +1,117 @@ +/* + * Copyright 2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot.impl; + +import org.gradle.internal.hash.ClassLoaderHierarchyHasher; +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.snapshot.ValueSnapshot; +import org.gradle.internal.snapshot.ValueSnapshotter; + +import javax.annotation.Nullable; + +/** + * Identifies a type in a classloader hierarchy. The type is identified by its name, + * the classloader hierarchy by its hash code. + */ +public abstract class ImplementationSnapshot implements ValueSnapshot { + private static final String GENERATED_LAMBDA_CLASS_SUFFIX = "$$Lambda$"; + public enum UnknownReason { + LAMBDA( + "was implemented by the Java lambda '%s'.", + "Using Java lambdas is not supported as task inputs.", + "Use an (anonymous inner) class instead."), + UNKNOWN_CLASSLOADER( + "was loaded with an unknown classloader (class '%s').", + "Gradle cannot track the implementation for classes loaded with an unknown classloader.", + "Load your class by using one of Gradle's built-in ways." + ); + + private final String descriptionTemplate; + private final String reason; + private final String solution; + + UnknownReason(String descriptionTemplate, String reason, String solution) { + this.descriptionTemplate = descriptionTemplate; + this.reason = reason; + this.solution = solution; + } + + public String descriptionFor(ImplementationSnapshot implementationSnapshot) { + return String.format(descriptionTemplate, implementationSnapshot.getTypeName()); + } + + public String getReason() { + return reason; + } + + public String getSolution() { + return solution; + } + } + + private final String typeName; + + public static ImplementationSnapshot of(Class type, ClassLoaderHierarchyHasher classLoaderHasher) { + String className = type.getName(); + return of(className, classLoaderHasher.getClassLoaderHash(type.getClassLoader()), type.isSynthetic() && isLambdaClassName(className)); + } + + public static ImplementationSnapshot of(String className, @Nullable HashCode classLoaderHash) { + return of(className, classLoaderHash, isLambdaClassName(className)); + } + + private static ImplementationSnapshot of(String typeName, @Nullable HashCode classLoaderHash, boolean lambda) { + if (classLoaderHash == null) { + return new UnknownClassloaderImplementationSnapshot(typeName); + } + if (lambda) { + return new LambdaImplementationSnapshot(typeName); + } + return new KnownImplementationSnapshot(typeName, classLoaderHash); + } + + private static boolean isLambdaClassName(String className) { + return className.contains(GENERATED_LAMBDA_CLASS_SUFFIX); + } + + protected ImplementationSnapshot(String typeName) { + this.typeName = typeName; + } + + public String getTypeName() { + return typeName; + } + + @Nullable + public abstract HashCode getClassLoaderHash(); + + public abstract boolean isUnknown(); + + @Nullable + public abstract UnknownReason getUnknownReason(); + + @Override + public ValueSnapshot snapshot(@Nullable Object value, ValueSnapshotter snapshotter) { + ValueSnapshot other = snapshotter.snapshot(value); + if (this.isSameSnapshot(other)) { + return this; + } + return other; + } + + protected abstract boolean isSameSnapshot(@Nullable Object o); +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/KnownImplementationSnapshot.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/KnownImplementationSnapshot.java new file mode 100644 index 00000000..41592331 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/KnownImplementationSnapshot.java @@ -0,0 +1,102 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot.impl; + +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.hash.Hasher; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public class KnownImplementationSnapshot extends ImplementationSnapshot { + private final HashCode classLoaderHash; + + public KnownImplementationSnapshot(String typeName, HashCode classLoaderHash) { + super(typeName); + this.classLoaderHash = classLoaderHash; + } + + @Override + public void appendToHasher(Hasher hasher) { + hasher.putString(ImplementationSnapshot.class.getName()); + hasher.putString(getTypeName()); + hasher.putHash(classLoaderHash); + } + + @Override + protected boolean isSameSnapshot(@Nullable Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + KnownImplementationSnapshot that = (KnownImplementationSnapshot) o; + + if (!getTypeName().equals(that.getTypeName())) { + return false; + } + return classLoaderHash.equals(that.classLoaderHash); + } + + @Nonnull + @Override + public HashCode getClassLoaderHash() { + return classLoaderHash; + } + + @Override + public boolean isUnknown() { + return false; + } + + @Override + @Nullable + public UnknownReason getUnknownReason() { + return null; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + KnownImplementationSnapshot that = (KnownImplementationSnapshot) o; + if (this == o) { + return true; + } + + + if (!getTypeName().equals(that.getTypeName())) { + return false; + } + return classLoaderHash.equals(that.classLoaderHash); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + result = 31 * result + classLoaderHash.hashCode(); + return result; + } + + @Override + public String toString() { + return getTypeName() + "@" + classLoaderHash; + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/LambdaImplementationSnapshot.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/LambdaImplementationSnapshot.java new file mode 100644 index 00000000..2dadef38 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/LambdaImplementationSnapshot.java @@ -0,0 +1,79 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot.impl; + +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.hash.Hasher; + +import javax.annotation.Nullable; + +public class LambdaImplementationSnapshot extends ImplementationSnapshot { + + public LambdaImplementationSnapshot(String typeName) { + super(typeName); + } + + @Override + public void appendToHasher(Hasher hasher) { + throw new RuntimeException("Cannot hash implementation of lambda " + getTypeName()); + } + + @Override + protected boolean isSameSnapshot(@Nullable Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + LambdaImplementationSnapshot that = (LambdaImplementationSnapshot) o; + + return getTypeName().equals(that.getTypeName()); + } + + @Override + public HashCode getClassLoaderHash() { + return null; + } + + @Override + public boolean isUnknown() { + return true; + } + + @Override + @Nullable + public UnknownReason getUnknownReason() { + return UnknownReason.LAMBDA; + } + + @Override + public boolean equals(Object o) { + return false; + } + + @Override + public int hashCode() { + return getTypeName().hashCode(); + } + + @Override + public String toString() { + return getTypeName(); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/MapEntrySnapshot.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/MapEntrySnapshot.java new file mode 100644 index 00000000..8217da23 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/MapEntrySnapshot.java @@ -0,0 +1,59 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot.impl; + +public class MapEntrySnapshot { + private final T key; + private final T value; + + public MapEntrySnapshot(T key, T value) { + this.key = key; + this.value = value; + } + + public T getKey() { + return key; + } + + public T getValue() { + return value; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + MapEntrySnapshot that = (MapEntrySnapshot) o; + + if (!key.equals(that.key)) { + return false; + } + return value.equals(that.value); + } + + @Override + public int hashCode() { + int result = key.hashCode(); + result = 31 * result + value.hashCode(); + return result; + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/UnknownClassloaderImplementationSnapshot.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/UnknownClassloaderImplementationSnapshot.java new file mode 100644 index 00000000..77539a9c --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/UnknownClassloaderImplementationSnapshot.java @@ -0,0 +1,79 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.snapshot.impl; + +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.hash.Hasher; + +import javax.annotation.Nullable; + +public class UnknownClassloaderImplementationSnapshot extends ImplementationSnapshot { + + public UnknownClassloaderImplementationSnapshot(String typeName) { + super(typeName); + } + + @Override + public void appendToHasher(Hasher hasher) { + throw new RuntimeException("Cannot hash implementation of class " + getTypeName() + " loaded by an unknown classloader"); + } + + @Override + protected boolean isSameSnapshot(@Nullable Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + UnknownClassloaderImplementationSnapshot that = (UnknownClassloaderImplementationSnapshot) o; + + return getTypeName().equals(that.getTypeName()); + } + + @Override + public HashCode getClassLoaderHash() { + return null; + } + + @Override + public boolean isUnknown() { + return true; + } + + @Override + @Nullable + public UnknownReason getUnknownReason() { + return UnknownReason.UNKNOWN_CLASSLOADER; + } + + @Override + public boolean equals(Object o) { + return false; + } + + @Override + public int hashCode() { + return getTypeName().hashCode(); + } + + @Override + public String toString() { + return getTypeName() + "@" + ""; + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/package-info.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/package-info.java new file mode 100644 index 00000000..a7d6fa0b --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/impl/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@NonNullApi +package org.gradle.internal.snapshot.impl; + +import org.gradle.api.NonNullApi; diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/package-info.java b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/package-info.java new file mode 100644 index 00000000..5607d067 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/snapshot/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright 2018 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@NonNullApi +package org.gradle.internal.snapshot; + +import org.gradle.api.NonNullApi; diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/FileSystemAccess.java b/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/FileSystemAccess.java new file mode 100644 index 00000000..2d5e34d1 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/FileSystemAccess.java @@ -0,0 +1,68 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.vfs; + +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.snapshot.FileSystemLocationSnapshot; +import org.gradle.internal.snapshot.SnapshottingFilter; + +import java.util.Optional; +import java.util.function.Consumer; +import java.util.function.Function; + +/** + * Provides access to snapshots of the content and metadata of the file system. + * + * The implementation will attempt to efficiently honour the queries, maintaining some or all state in-memory and dealing with concurrent access to the same parts of the file system. + * + * The file system access needs to be informed when some state on disk changes, so it does not become out of sync with the actual file system. + */ +public interface FileSystemAccess { + + /** + * Visits the hash of the content of the file only if the file is a regular file. + * + * @return the visitor function applied to the found snapshot. + */ + Optional readRegularFileContentHash(String location, Function visitor); + + /** + * Visits the hierarchy of files at the given location. + */ + T read(String location, Function visitor); + + /** + * Visits the hierarchy of files which match the filter at the given location. + * + * The consumer is only called if if something matches the filter. + */ + void read(String location, SnapshottingFilter filter, Consumer visitor); + + /** + * Runs an action which potentially writes to the given locations. + */ + void write(Iterable locations, Runnable action); + + /** + * Updates the cached state at the location with the snapshot. + */ + void record(FileSystemLocationSnapshot snapshot); + + interface WriteListener { + void locationsWritten(Iterable locations); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/VirtualFileSystem.java b/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/VirtualFileSystem.java new file mode 100644 index 00000000..8624dc28 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/VirtualFileSystem.java @@ -0,0 +1,51 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.vfs; + +import org.gradle.internal.snapshot.FileSystemLocationSnapshot; +import org.gradle.internal.snapshot.MetadataSnapshot; + +import java.util.Optional; + +public interface VirtualFileSystem { + + /** + * Returns the snapshot stored at the absolute path. + */ + Optional getSnapshot(String absolutePath); + + /** + * Returns the metadata stored at the absolute path. + */ + Optional getMetadata(String absolutePath); + + /** + * Adds the information of the snapshot at the absolute path to the VFS. + */ + void store(String absolutePath, FileSystemLocationSnapshot snapshot); + + /** + * Removes any information at the absolute paths from the VFS. + */ + void invalidate(Iterable locations); + + /** + * Removes any information from the VFS. + */ + void invalidateAll(); + +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/impl/AbstractVirtualFileSystem.java b/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/impl/AbstractVirtualFileSystem.java new file mode 100644 index 00000000..101b090e --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/impl/AbstractVirtualFileSystem.java @@ -0,0 +1,82 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.vfs.impl; + +import org.gradle.internal.snapshot.FileSystemLocationSnapshot; +import org.gradle.internal.snapshot.MetadataSnapshot; +import org.gradle.internal.snapshot.SnapshotHierarchy; +import org.gradle.internal.vfs.VirtualFileSystem; + +import java.util.Optional; + +public abstract class AbstractVirtualFileSystem implements VirtualFileSystem { + + protected final VfsRootReference rootReference; + + protected AbstractVirtualFileSystem(VfsRootReference rootReference) { + this.rootReference = rootReference; + } + + @Override + public Optional getSnapshot(String absolutePath) { + return rootReference.getRoot().getSnapshot(absolutePath); + } + + @Override + public Optional getMetadata(String absolutePath) { + return rootReference.getRoot().getMetadata(absolutePath); + } + + @Override + public void store(String absolutePath, FileSystemLocationSnapshot snapshot) { + rootReference.update(root -> updateNotifyingListeners(diffListener -> root.store(absolutePath, snapshot, diffListener))); + } + + @Override + public void invalidate(Iterable locations) { + rootReference.update(root -> { + SnapshotHierarchy result = root; + for (String location : locations) { + SnapshotHierarchy currentRoot = result; + result = updateNotifyingListeners(diffListener -> currentRoot.invalidate(location, diffListener)); + } + return result; + }); + } + + @Override + public void invalidateAll() { + rootReference.update(root -> updateNotifyingListeners(diffListener -> { + root.visitSnapshotRoots(diffListener::nodeRemoved); + return root.empty(); + })); + } + + /** + * Runs a single update on a {@link SnapshotHierarchy} and notifies the currently active listeners after the update. + */ + protected abstract SnapshotHierarchy updateNotifyingListeners(UpdateFunction updateFunction); + + public interface UpdateFunction { + /** + * Runs a single update on a {@link SnapshotHierarchy}, notifying the diffListener about changes. + * + * @return updated ${@link SnapshotHierarchy}. + */ + SnapshotHierarchy update(SnapshotHierarchy.NodeDiffListener diffListener); + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/impl/DefaultFileSystemAccess.java b/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/impl/DefaultFileSystemAccess.java new file mode 100644 index 00000000..bd2a6fcf --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/impl/DefaultFileSystemAccess.java @@ -0,0 +1,235 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.vfs.impl; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Interner; +import com.google.common.util.concurrent.Striped; +import org.gradle.internal.file.FileMetadata; +import org.gradle.internal.file.FileMetadata.AccessType; +import org.gradle.internal.file.FileType; +import org.gradle.internal.file.Stat; +import org.gradle.internal.hash.FileHasher; +import org.gradle.internal.hash.HashCode; +import org.gradle.internal.snapshot.FileSystemLocationSnapshot; +import org.gradle.internal.snapshot.FileSystemSnapshot; +import org.gradle.internal.snapshot.MissingFileSnapshot; +import org.gradle.internal.snapshot.RegularFileSnapshot; +import org.gradle.internal.snapshot.SnapshottingFilter; +import org.gradle.internal.snapshot.impl.DirectorySnapshotter; +import org.gradle.internal.snapshot.impl.DirectorySnapshotterStatistics; +import org.gradle.internal.snapshot.impl.FileSystemSnapshotFilter; +import org.gradle.internal.vfs.FileSystemAccess; +import org.gradle.internal.vfs.VirtualFileSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Lock; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; + +public class DefaultFileSystemAccess implements FileSystemAccess { + private static final Logger LOGGER = LoggerFactory.getLogger(DefaultFileSystemAccess.class); + + private final VirtualFileSystem virtualFileSystem; + private final Stat stat; + private final Interner stringInterner; + private final WriteListener writeListener; + private final DirectorySnapshotterStatistics.Collector statisticsCollector; + private ImmutableList defaultExcludes; + private DirectorySnapshotter directorySnapshotter; + private final FileHasher hasher; + private final StripedProducerGuard producingSnapshots = new StripedProducerGuard<>(); + + public DefaultFileSystemAccess( + FileHasher hasher, + Interner stringInterner, + Stat stat, + VirtualFileSystem virtualFileSystem, + WriteListener writeListener, + DirectorySnapshotterStatistics.Collector statisticsCollector, + String... defaultExcludes + ) { + this.stringInterner = stringInterner; + this.stat = stat; + this.writeListener = writeListener; + this.statisticsCollector = statisticsCollector; + this.defaultExcludes = ImmutableList.copyOf(defaultExcludes); + this.directorySnapshotter = new DirectorySnapshotter(hasher, stringInterner, this.defaultExcludes, statisticsCollector); + this.hasher = hasher; + this.virtualFileSystem = virtualFileSystem; + } + + @Override + public T read(String location, Function visitor) { + return visitor.apply(readLocation(location)); + } + + @Override + public Optional readRegularFileContentHash(String location, Function visitor) { + return virtualFileSystem.getMetadata(location) + .>flatMap(snapshot -> { + if (snapshot.getType() != FileType.RegularFile) { + return Optional.of(Optional.empty()); + } + if (snapshot instanceof FileSystemLocationSnapshot) { + return Optional.of(Optional.of(((FileSystemLocationSnapshot) snapshot).getHash())); + } + return Optional.empty(); + }) + .orElseGet(() -> { + File file = new File(location); + FileMetadata fileMetadata = this.stat.stat(file); + if (fileMetadata.getType() == FileType.Missing) { + storeMetadataForMissingFile(location, fileMetadata.getAccessType()); + } + if (fileMetadata.getType() != FileType.RegularFile) { + return Optional.empty(); + } + HashCode hash = producingSnapshots.guardByKey(location, + () -> virtualFileSystem.getSnapshot(location) + .orElseGet(() -> { + HashCode hashCode = hasher.hash(file, fileMetadata.getLength(), fileMetadata.getLastModified()); + RegularFileSnapshot snapshot = new RegularFileSnapshot(location, file.getName(), hashCode, fileMetadata); + virtualFileSystem.store(snapshot.getAbsolutePath(), snapshot); + return snapshot; + }).getHash()); + return Optional.of(hash); + }) + .map(visitor); + } + + private void storeMetadataForMissingFile(String location, AccessType accessType) { + virtualFileSystem.store(location, new MissingFileSnapshot(location, accessType)); + } + + @Override + public void read(String location, SnapshottingFilter filter, Consumer visitor) { + if (filter.isEmpty()) { + visitor.accept(readLocation(location)); + } else { + FileSystemSnapshot filteredSnapshot = readSnapshotFromLocation(location, + snapshot -> FileSystemSnapshotFilter.filterSnapshot(filter.getAsSnapshotPredicate(), snapshot), + () -> { + FileSystemLocationSnapshot snapshot = snapshot(location, filter); + return snapshot.getType() == FileType.Directory + // Directory snapshots have been filtered while walking the file system + ? snapshot + : FileSystemSnapshotFilter.filterSnapshot(filter.getAsSnapshotPredicate(), snapshot); + }); + + if (filteredSnapshot instanceof FileSystemLocationSnapshot) { + visitor.accept((FileSystemLocationSnapshot) filteredSnapshot); + } + } + } + + private FileSystemLocationSnapshot snapshot(String location, SnapshottingFilter filter) { + File file = new File(location); + FileMetadata fileMetadata = this.stat.stat(file); + switch (fileMetadata.getType()) { + case RegularFile: + HashCode hash = hasher.hash(file, fileMetadata.getLength(), fileMetadata.getLastModified()); + RegularFileSnapshot regularFileSnapshot = new RegularFileSnapshot(location, file.getName(), hash, fileMetadata); + virtualFileSystem.store(regularFileSnapshot.getAbsolutePath(), regularFileSnapshot); + return regularFileSnapshot; + case Missing: + MissingFileSnapshot missingFileSnapshot = new MissingFileSnapshot(location, fileMetadata.getAccessType()); + virtualFileSystem.store(missingFileSnapshot.getAbsolutePath(), missingFileSnapshot); + return missingFileSnapshot; + case Directory: + AtomicBoolean hasBeenFiltered = new AtomicBoolean(false); + FileSystemLocationSnapshot directorySnapshot = directorySnapshotter.snapshot(location, filter.isEmpty() ? null : filter.getAsDirectoryWalkerPredicate(), hasBeenFiltered); + if (!hasBeenFiltered.get()) { + virtualFileSystem.store(directorySnapshot.getAbsolutePath(), directorySnapshot); + } + return directorySnapshot; + default: + throw new UnsupportedOperationException(); + } + } + + private FileSystemLocationSnapshot readLocation(String location) { + return readSnapshotFromLocation(location, () -> snapshot(location, SnapshottingFilter.EMPTY)); + } + + private FileSystemLocationSnapshot readSnapshotFromLocation( + String location, + Supplier readFromDisk + ) { + return readSnapshotFromLocation( + location, + Function.identity(), + readFromDisk + ); + } + + private T readSnapshotFromLocation( + String location, + Function snapshotProcessor, + Supplier readFromDisk + ) { + return virtualFileSystem.getSnapshot(location) + .map(snapshotProcessor) + // Avoid snapshotting the same location at the same time + .orElseGet(() -> producingSnapshots.guardByKey(location, + () -> virtualFileSystem.getSnapshot(location) + .map(snapshotProcessor) + .orElseGet(readFromDisk) + )); + } + + @Override + public void write(Iterable locations, Runnable action) { + writeListener.locationsWritten(locations); + virtualFileSystem.invalidate(locations); + action.run(); + } + + @Override + public void record(FileSystemLocationSnapshot snapshot) { + virtualFileSystem.store(snapshot.getAbsolutePath(), snapshot); + } + + private static class StripedProducerGuard { + private final Striped locks = Striped.lock(Runtime.getRuntime().availableProcessors() * 4); + + public V guardByKey(T key, Supplier supplier) { + Lock lock = locks.get(key); + try { + lock.lock(); + return supplier.get(); + } finally { + lock.unlock(); + } + } + } + + public void updateDefaultExcludes(String... newDefaultExcludesArgs) { + ImmutableList newDefaultExcludes = ImmutableList.copyOf(newDefaultExcludesArgs); + if (!defaultExcludes.equals(newDefaultExcludes)) { + LOGGER.debug("Default excludes changes from {} to {}", defaultExcludes, newDefaultExcludes); + defaultExcludes = newDefaultExcludes; + directorySnapshotter = new DirectorySnapshotter(hasher, stringInterner, newDefaultExcludes, statisticsCollector); + virtualFileSystem.invalidateAll(); + } + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/impl/DefaultSnapshotHierarchy.java b/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/impl/DefaultSnapshotHierarchy.java new file mode 100644 index 00000000..0718d059 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/impl/DefaultSnapshotHierarchy.java @@ -0,0 +1,165 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.vfs.impl; + +import com.google.common.annotations.VisibleForTesting; +import org.gradle.internal.file.FileType; +import org.gradle.internal.snapshot.CaseSensitivity; +import org.gradle.internal.snapshot.FileSystemNode; +import org.gradle.internal.snapshot.MetadataSnapshot; +import org.gradle.internal.snapshot.PartialDirectoryNode; +import org.gradle.internal.snapshot.ReadOnlyFileSystemNode; +import org.gradle.internal.snapshot.SingletonChildMap; +import org.gradle.internal.snapshot.SnapshotHierarchy; +import org.gradle.internal.snapshot.UnknownFileSystemNode; +import org.gradle.internal.snapshot.VfsRelativePath; + +import java.util.Optional; + +public class DefaultSnapshotHierarchy implements SnapshotHierarchy { + + private final CaseSensitivity caseSensitivity; + @VisibleForTesting + final FileSystemNode rootNode; + + public static SnapshotHierarchy from(FileSystemNode rootNode, CaseSensitivity caseSensitivity) { + return new DefaultSnapshotHierarchy(rootNode, caseSensitivity); + } + + private DefaultSnapshotHierarchy(FileSystemNode rootNode, CaseSensitivity caseSensitivity) { + this.caseSensitivity = caseSensitivity; + this.rootNode = rootNode; + } + + public static SnapshotHierarchy empty(CaseSensitivity caseSensitivity) { + switch (caseSensitivity) { + case CASE_SENSITIVE: + return EmptySnapshotHierarchy.CASE_SENSITIVE; + case CASE_INSENSITIVE: + return EmptySnapshotHierarchy.CASE_INSENSITIVE; + default: + throw new AssertionError("Unknown case sensitivity: " + caseSensitivity); + } + } + + @Override + public Optional getMetadata(String absolutePath) { + VfsRelativePath relativePath = VfsRelativePath.of(absolutePath); + if (relativePath.length() == 0) { + return rootNode.getSnapshot(); + } + return rootNode.getSnapshot(relativePath, caseSensitivity); + } + + @Override + public boolean hasDescendantsUnder(String absolutePath) { + return getNode(absolutePath).hasDescendants(); + } + + @Override + public SnapshotHierarchy store(String absolutePath, MetadataSnapshot snapshot, NodeDiffListener diffListener) { + VfsRelativePath relativePath = VfsRelativePath.of(absolutePath); + if (relativePath.length() == 0) { + return new DefaultSnapshotHierarchy(snapshot.asFileSystemNode(), caseSensitivity); + } + return new DefaultSnapshotHierarchy( + rootNode.store(relativePath, caseSensitivity, snapshot, diffListener), + caseSensitivity + ); + } + + @Override + public SnapshotHierarchy invalidate(String absolutePath, NodeDiffListener diffListener) { + VfsRelativePath relativePath = VfsRelativePath.of(absolutePath); + if (relativePath.length() == 0) { + diffListener.nodeRemoved(rootNode); + return empty(); + } + return rootNode.invalidate(relativePath, caseSensitivity, diffListener) + .map(it -> (it == rootNode) ? this : new DefaultSnapshotHierarchy(it, caseSensitivity)) + .orElseGet(() -> empty(caseSensitivity)); + } + + @Override + public SnapshotHierarchy empty() { + return empty(caseSensitivity); + } + + @Override + public void visitSnapshotRoots(SnapshotVisitor snapshotVisitor) { + rootNode.accept(snapshotVisitor); + } + + @Override + public void visitSnapshotRoots(String absolutePath, SnapshotVisitor snapshotVisitor) { + getNode(absolutePath).accept(snapshotVisitor); + } + + private ReadOnlyFileSystemNode getNode(String absolutePath) { + VfsRelativePath relativePath = VfsRelativePath.of(absolutePath); + return (relativePath.length() == 0) ? rootNode : rootNode.getNode(relativePath, caseSensitivity); + } + + private enum EmptySnapshotHierarchy implements SnapshotHierarchy { + CASE_SENSITIVE(CaseSensitivity.CASE_SENSITIVE), + CASE_INSENSITIVE(CaseSensitivity.CASE_INSENSITIVE); + + private final CaseSensitivity caseSensitivity; + + EmptySnapshotHierarchy(CaseSensitivity caseInsensitive) { + this.caseSensitivity = caseInsensitive; + } + + @Override + public Optional getMetadata(String absolutePath) { + return Optional.empty(); + } + + @Override + public boolean hasDescendantsUnder(String absolutePath) { + return false; + } + + @Override + public SnapshotHierarchy store(String absolutePath, MetadataSnapshot snapshot, NodeDiffListener diffListener) { + VfsRelativePath relativePath = VfsRelativePath.of(absolutePath); + String childPath = relativePath.getAsString(); + SingletonChildMap children = new SingletonChildMap<>(childPath, snapshot.asFileSystemNode()); + FileSystemNode rootNode = snapshot.getType() == FileType.Missing + ? new UnknownFileSystemNode(children) + : new PartialDirectoryNode(children); + diffListener.nodeAdded(rootNode); + return from(rootNode, caseSensitivity); + } + + @Override + public SnapshotHierarchy invalidate(String absolutePath, NodeDiffListener diffListener) { + return this; + } + + @Override + public SnapshotHierarchy empty() { + return this; + } + + @Override + public void visitSnapshotRoots(SnapshotVisitor snapshotVisitor) {} + + @Override + public void visitSnapshotRoots(String absolutePath, SnapshotVisitor snapshotVisitor) {} + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/impl/VfsRootReference.java b/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/impl/VfsRootReference.java new file mode 100644 index 00000000..700579f7 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/impl/VfsRootReference.java @@ -0,0 +1,45 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gradle.internal.vfs.impl; + +import org.gradle.internal.snapshot.SnapshotHierarchy; + +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.UnaryOperator; + +public class VfsRootReference { + private volatile SnapshotHierarchy root; + private final ReentrantLock updateLock = new ReentrantLock(); + + public SnapshotHierarchy getRoot() { + return root; + } + + public VfsRootReference(SnapshotHierarchy root) { + this.root = root; + } + + public void update(UnaryOperator updateFunction) { + updateLock.lock(); + try { + SnapshotHierarchy currentRoot = root; + root = updateFunction.apply(currentRoot); + } finally { + updateLock.unlock(); + } + } +} diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/impl/package-info.java b/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/impl/package-info.java new file mode 100644 index 00000000..b95cd33c --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/impl/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +@NonNullApi +package org.gradle.internal.vfs.impl; + +import org.gradle.api.NonNullApi; diff --git a/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/package-info.java b/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/package-info.java new file mode 100644 index 00000000..9bbca839 --- /dev/null +++ b/subprojects/snapshots/src/main/java/org/gradle/internal/vfs/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@NonNullApi +package org.gradle.internal.vfs; + +import org.gradle.api.NonNullApi; diff --git a/template/.gitignore b/template/.gitignore new file mode 100644 index 00000000..42afabfd --- /dev/null +++ b/template/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/template/build.gradle.kts b/template/build.gradle.kts new file mode 100644 index 00000000..f48d0c38 --- /dev/null +++ b/template/build.gradle.kts @@ -0,0 +1,12 @@ +plugins { + id("java-library") +} + +java { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} + +dependencies { + +} \ No newline at end of file