-
Notifications
You must be signed in to change notification settings - Fork 304
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This CL implements the caching and tracking of artifacts produced by the build, but does not include the corresponding update to the project proto which will follow in later CLs. As such, it is not yet working. Modify `JavaArtifactInfo` to use a new `BuildArtifact` class in place of paths for built artifacts. This allows the digest and origin target to be included here. Populate the digests in `NewArtifactTracker` using information derived from the BES. This allows the artifacts to be requested from the `BuildArtifactCache` later on. The existing `ArtifactTrackerImpl` does not need the digests so we create temporary support for omitting them. To persist the artifact tracker state to disk, we use a new proto definition rather than re-using the format produced by the aspect. This allows extra information to be included (e.g. the digests), and also allows the persisted state to change in future without impacting the aspect code. Add serialization and deserialization code to support this. PiperOrigin-RevId: 613190595
- Loading branch information
1 parent
41d78a5
commit da2daad
Showing
16 changed files
with
751 additions
and
56 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
86 changes: 86 additions & 0 deletions
86
querysync/java/com/google/idea/blaze/qsync/artifacts/BuildArtifact.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| /* | ||
| * Copyright 2024 The Bazel Authors. All rights reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package com.google.idea.blaze.qsync.artifacts; | ||
|
|
||
| import com.google.auto.value.AutoValue; | ||
| import com.google.common.base.Preconditions; | ||
| import com.google.common.io.ByteSource; | ||
| import com.google.common.util.concurrent.Uninterruptibles; | ||
| import com.google.idea.blaze.common.Label; | ||
| import com.google.idea.blaze.common.artifact.BuildArtifactCache; | ||
| import com.google.idea.blaze.exception.BuildException; | ||
| import java.nio.file.Path; | ||
| import java.util.concurrent.ExecutionException; | ||
| import java.util.function.Function; | ||
|
|
||
| /** | ||
| * An artifact produced by a build. | ||
| * | ||
| * <p>This includes the digest of the artifact as indicated by bazel, it's output path and the | ||
| * target that produced it. | ||
| * | ||
| * <p>Note, for the old {@link com.google.idea.blaze.base.qsync.cache.ArtifactTrackerImpl} codepaths | ||
| * the digest will not be populated. | ||
| */ | ||
| @AutoValue | ||
| public abstract class BuildArtifact { | ||
|
|
||
| /** | ||
| * Special case to support legacy codepaths that don't use digests in here | ||
| * | ||
| * <p>TODO(b/323346056) remove this when the old codepaths are deleted. | ||
| */ | ||
| public static final Function<Path, String> NO_DIGESTS = p -> ""; | ||
|
|
||
| public abstract String digest(); | ||
|
|
||
| public abstract Path path(); | ||
|
|
||
| public abstract Label target(); | ||
|
|
||
| public static BuildArtifact create(String digest, Path path, Label target) { | ||
| return new AutoValue_BuildArtifact(digest, path, target); | ||
| } | ||
|
|
||
| public static BuildArtifact create(Path path, Label target, Function<Path, String> digestMap) { | ||
| String digest = | ||
| Preconditions.checkNotNull(digestMap.apply(path), "No digest for %s from %s", path, target); | ||
| if (digestMap != NO_DIGESTS) { | ||
| Preconditions.checkState(!digest.isEmpty(), "Empty digest for %s from %s", path, target); | ||
| } | ||
| return create(digest, path, target); | ||
| } | ||
|
|
||
| public ByteSource blockingGetFrom(BuildArtifactCache cache) throws BuildException { | ||
| try { | ||
| return Uninterruptibles.getUninterruptibly( | ||
| cache | ||
| .get(digest()) | ||
| .orElseThrow(() -> new BuildException("Artifact %s missing from the cache" + this))); | ||
| } catch (ExecutionException e) { | ||
| throw new BuildException("Failed to get artifact " + this, e); | ||
| } | ||
| } | ||
|
|
||
| public String getExtension() { | ||
| String fileName = path().getFileName().toString(); | ||
| int lastDot = fileName.lastIndexOf('.'); | ||
| if (lastDot == -1) { | ||
| return ""; | ||
| } | ||
| return fileName.substring(lastDot + 1); | ||
| } | ||
| } |
100 changes: 100 additions & 0 deletions
100
querysync/java/com/google/idea/blaze/qsync/deps/ArtifactTrackerStateDeserializer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| /* | ||
| * Copyright 2024 The Bazel Authors. All rights reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package com.google.idea.blaze.qsync.deps; | ||
|
|
||
| import static com.google.common.collect.ImmutableList.toImmutableList; | ||
| import static com.google.common.collect.ImmutableSet.toImmutableSet; | ||
|
|
||
| import com.google.common.base.Preconditions; | ||
| import com.google.common.collect.ImmutableList; | ||
| import com.google.common.collect.ImmutableMap; | ||
| import com.google.common.collect.Maps; | ||
| import com.google.idea.blaze.common.Label; | ||
| import com.google.idea.blaze.common.vcs.VcsState; | ||
| import com.google.idea.blaze.qsync.artifacts.BuildArtifact; | ||
| import com.google.idea.blaze.qsync.java.ArtifactTrackerProto; | ||
| import com.google.idea.blaze.qsync.java.JavaArtifactInfo; | ||
| import com.google.idea.blaze.qsync.java.cc.CcCompilationInfoOuterClass.CcToolchainInfo; | ||
| import com.google.idea.blaze.qsync.project.SnapshotDeserializer; | ||
| import java.nio.file.Path; | ||
| import java.time.Instant; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
|
|
||
| /** Deserializes {@link NewArtifactTracker} state from a proto. */ | ||
| public class ArtifactTrackerStateDeserializer { | ||
|
|
||
| private final ImmutableMap.Builder<Label, TargetBuildInfo> map = ImmutableMap.builder(); | ||
| private final Map<String, DependencyBuildContext> buildContexts = Maps.newHashMap(); | ||
| private final ImmutableMap.Builder<String, CcToolchainInfo> ccToolchainMap = | ||
| ImmutableMap.builder(); | ||
|
|
||
| public void visit(ArtifactTrackerProto.ArtifactTrackerState proto) { | ||
| proto.getBuildContextsList().forEach(this::visitBuildContext); | ||
| proto.getBuiltDepsMap().entrySet().forEach(this::visitTargetBuildInfo); | ||
| } | ||
|
|
||
| public ImmutableMap<Label, TargetBuildInfo> getBuiltDepsMap() { | ||
| return map.buildOrThrow(); | ||
| } | ||
|
|
||
| private void visitBuildContext(ArtifactTrackerProto.BuildContext buildContext) { | ||
| Optional<VcsState> vcsState = Optional.empty(); | ||
| if (buildContext.hasVcsState()) { | ||
| vcsState = Optional.of(SnapshotDeserializer.convertVcsState(buildContext.getVcsState())); | ||
| } | ||
| buildContexts.put( | ||
| buildContext.getBuildId(), | ||
| DependencyBuildContext.create( | ||
| buildContext.getBuildId(), | ||
| Instant.ofEpochMilli(buildContext.getStartTimeMillis()), | ||
| vcsState)); | ||
| } | ||
|
|
||
| private void visitTargetBuildInfo(Map.Entry<String, ArtifactTrackerProto.TargetBuildInfo> entry) { | ||
| ArtifactTrackerProto.TargetBuildInfo proto = entry.getValue(); | ||
| DependencyBuildContext buildContext = | ||
| Preconditions.checkNotNull(buildContexts.get(proto.getBuildId())); | ||
| Label owner = Label.of(entry.getKey()); | ||
| if (proto.hasJavaArtifacts()) { | ||
| map.put( | ||
| owner, | ||
| TargetBuildInfo.forJavaTarget( | ||
| convertJavaArtifactInfo(owner, proto.getJavaArtifacts()), buildContext)); | ||
| } | ||
| } | ||
|
|
||
| private JavaArtifactInfo convertJavaArtifactInfo( | ||
| Label owner, ArtifactTrackerProto.JavaArtifacts proto) { | ||
| return JavaArtifactInfo.builder() | ||
| .setLabel(owner) | ||
| .setJars(toArtifactList(proto.getJarsList(), owner)) | ||
| .setIdeAars(toArtifactList(proto.getIdeAarsList(), owner)) | ||
| .setGenSrcs(toArtifactList(proto.getGenSrcsList(), owner)) | ||
| .setSources(proto.getSourcesList().stream().map(Path::of).collect(toImmutableSet())) | ||
| .setSrcJars(proto.getSrcJarsList().stream().map(Path::of).collect(toImmutableSet())) | ||
| .setAndroidResourcesPackage(proto.getAndroidResourcesPackage()) | ||
| .build(); | ||
| } | ||
|
|
||
| private ImmutableList<BuildArtifact> toArtifactList( | ||
| List<ArtifactTrackerProto.Artifact> protos, Label owner) { | ||
| return protos.stream() | ||
| .map(a -> BuildArtifact.create(a.getDigest(), Path.of(a.getPath()), owner)) | ||
| .collect(toImmutableList()); | ||
| } | ||
| } |
99 changes: 99 additions & 0 deletions
99
querysync/java/com/google/idea/blaze/qsync/deps/ArtifactTrackerStateSerializer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| /* | ||
| * Copyright 2024 The Bazel Authors. All rights reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
| package com.google.idea.blaze.qsync.deps; | ||
|
|
||
| import static com.google.common.collect.ImmutableList.toImmutableList; | ||
|
|
||
| import com.google.common.collect.ImmutableList; | ||
| import com.google.common.collect.Sets; | ||
| import com.google.errorprone.annotations.CanIgnoreReturnValue; | ||
| import com.google.idea.blaze.common.Label; | ||
| import com.google.idea.blaze.qsync.artifacts.BuildArtifact; | ||
| import com.google.idea.blaze.qsync.java.ArtifactTrackerProto; | ||
| import com.google.idea.blaze.qsync.java.ArtifactTrackerProto.Artifact; | ||
| import com.google.idea.blaze.qsync.java.ArtifactTrackerProto.ArtifactTrackerState; | ||
| import com.google.idea.blaze.qsync.java.ArtifactTrackerProto.BuildContext; | ||
| import com.google.idea.blaze.qsync.java.JavaArtifactInfo; | ||
| import com.google.idea.blaze.qsync.project.SnapshotSerializer; | ||
| import java.nio.file.Path; | ||
| import java.util.Map; | ||
| import java.util.Set; | ||
|
|
||
| /** Serializes {@link NewArtifactTracker} state to a proto. */ | ||
| public class ArtifactTrackerStateSerializer { | ||
|
|
||
| public static final int VERSION = 1; | ||
|
|
||
| private final ArtifactTrackerProto.ArtifactTrackerState.Builder proto = | ||
| ArtifactTrackerProto.ArtifactTrackerState.newBuilder().setVersion(VERSION); | ||
| private final Set<String> buildIdsSeen = Sets.newHashSet(); | ||
|
|
||
| @CanIgnoreReturnValue | ||
| public ArtifactTrackerStateSerializer visitDepsMap(Map<Label, TargetBuildInfo> builtDeps) { | ||
| builtDeps.entrySet().forEach(e -> visitTargetBuildInfo(e.getKey(), e.getValue())); | ||
| return this; | ||
| } | ||
|
|
||
| public ArtifactTrackerState toProto() { | ||
| return proto.build(); | ||
| } | ||
|
|
||
| private void visitTargetBuildInfo(Label target, TargetBuildInfo targetBuildInfo) { | ||
| visitBuildContext(targetBuildInfo.buildContext()); | ||
|
|
||
| ArtifactTrackerProto.TargetBuildInfo.Builder builder = | ||
| ArtifactTrackerProto.TargetBuildInfo.newBuilder(); | ||
| builder.setBuildId(targetBuildInfo.buildContext().buildId()); | ||
| targetBuildInfo.javaInfo().ifPresent(ji -> visitJavaInfo(ji, builder)); | ||
| proto.putBuiltDeps(target.toString(), builder.build()); | ||
| } | ||
|
|
||
| private void visitBuildContext(DependencyBuildContext buildContext) { | ||
| if (buildIdsSeen.add(buildContext.buildId())) { | ||
| BuildContext.Builder builder = | ||
| proto | ||
| .addBuildContextsBuilder() | ||
| .setStartTimeMillis(buildContext.startTime().toEpochMilli()) | ||
| .setBuildId(buildContext.buildId()); | ||
| buildContext | ||
| .vcsState() | ||
| .ifPresent(vcs -> SnapshotSerializer.visitVcsState(vcs, builder.getVcsStateBuilder())); | ||
| } | ||
| } | ||
|
|
||
| private void visitJavaInfo( | ||
| JavaArtifactInfo javaInfo, ArtifactTrackerProto.TargetBuildInfo.Builder builder) { | ||
| builder | ||
| .getJavaArtifactsBuilder() | ||
| .addAllGenSrcs(toProtos(javaInfo.genSrcs())) | ||
| .addAllIdeAars(toProtos(javaInfo.ideAars())) | ||
| .addAllJars(toProtos(javaInfo.jars())) | ||
| .addAllSources(javaInfo.sources().stream().map(Path::toString).collect(toImmutableList())) | ||
| .addAllSrcJars(javaInfo.srcJars().stream().map(Path::toString).collect(toImmutableList())) | ||
| .setAndroidResourcesPackage(javaInfo.androidResourcesPackage()); | ||
| } | ||
|
|
||
| private ImmutableList<Artifact> toProtos(ImmutableList<BuildArtifact> artifacts) { | ||
| return artifacts.stream() | ||
| .map( | ||
| artifact -> | ||
| ArtifactTrackerProto.Artifact.newBuilder() | ||
| .setDigest(artifact.digest()) | ||
| .setPath(artifact.path().toString()) | ||
| .build()) | ||
| .collect(toImmutableList()); | ||
| } | ||
| } |
Oops, something went wrong.