From e9a038343711bec1550564ee706671c57750e765 Mon Sep 17 00:00:00 2001 From: Brentley Jones Date: Tue, 25 Jul 2023 03:48:22 -0500 Subject: [PATCH 001/125] [6.4.0] Support multiple remote execution digest functions (#19042) * Support multiple remote execution digest functions The following PR on the remote-apis side adds support for remote execution services to announce support for multiple digest functions: https://github.com/bazelbuild/remote-apis/pull/236 This makes migrating from one digest function to the other more graceful. This change extends Bazel's server capabilities checking code to take the new field in the execution capabilities into account. Partial commit for third_party/*, see #16791. Signed-off-by: Sunil Gowroji (cherry picked from commit d0cba5507fcb5d636b1a9a3b1f58cf63314781c0) * Support multiple remote execution digest functions The following PR on the remote-apis side adds support for remote execution services to announce support for multiple digest functions: https://github.com/bazelbuild/remote-apis/pull/236 This makes migrating from one digest function to the other more graceful. This change extends Bazel's server capabilities checking code to take the new field in the execution capabilities into account. Closes #16791. PiperOrigin-RevId: 517084447 Change-Id: I72afce6c1fae9e624f9e7ed1936c744ae9c81280 (cherry picked from commit f8c88752acf6da4448df52d87e9fee54ca9d826c) --------- Co-authored-by: Ed Schouten --- .../lib/remote/RemoteServerCapabilities.java | 23 ++- .../remote/RemoteServerCapabilitiesTest.java | 25 +++ .../execution/v2/remote_execution.proto | 188 ++++++++++++++++-- 3 files changed, 215 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteServerCapabilities.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteServerCapabilities.java index 0afb9894217f3c..a69d4f60e1faea 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteServerCapabilities.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteServerCapabilities.java @@ -201,17 +201,24 @@ public static ClientServerCompatibilityStatus checkClientServerCompatibility( return result.build(); // No point checking other execution fields. } - // Check execution digest function. - if (execCap.getDigestFunction() == DigestFunction.Value.UNKNOWN) { - // Server side error -- this is not supposed to happen. - result.addError("Remote server error: UNKNOWN execution digest function."); - } - if (execCap.getDigestFunction() != digestFunction) { + // Check execution digest function. The protocol only later added + // support for multiple digest functions for remote execution, so + // check both the singular and repeated field. + if (execCap.getDigestFunctionsList().isEmpty() + && execCap.getDigestFunction() != DigestFunction.Value.UNKNOWN) { + if (execCap.getDigestFunction() != digestFunction) { + result.addError( + String.format( + "Cannot use hash function %s with remote execution. " + + "Server supported function is %s", + digestFunction, execCap.getDigestFunction())); + } + } else if (!execCap.getDigestFunctionsList().contains(digestFunction)) { result.addError( String.format( "Cannot use hash function %s with remote execution. " - + "Server supported function is %s", - digestFunction, execCap.getDigestFunction())); + + "Server supported functions are: %s", + digestFunction, execCap.getDigestFunctionsList())); } // Check execution priority is in the supported range. diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteServerCapabilitiesTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteServerCapabilitiesTest.java index a91bd7833e5b0c..aa2b21c7f286c4 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteServerCapabilitiesTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteServerCapabilitiesTest.java @@ -582,6 +582,31 @@ public void testCheckClientServerCompatibility_executionCapsOnly() throws Except assertThat(st.isOk()).isTrue(); } + @Test + public void testCheckClientServerCompatibility_executionCapsDigestFunctionsList() + throws Exception { + ServerCapabilities caps = + ServerCapabilities.newBuilder() + .setLowApiVersion(ApiVersion.current.toSemVer()) + .setHighApiVersion(ApiVersion.current.toSemVer()) + .setExecutionCapabilities( + ExecutionCapabilities.newBuilder() + .addDigestFunctions(DigestFunction.Value.MD5) + .addDigestFunctions(DigestFunction.Value.SHA256) + .setExecEnabled(true) + .build()) + .build(); + RemoteOptions remoteOptions = Options.getDefaults(RemoteOptions.class); + remoteOptions.remoteExecutor = "server:port"; + RemoteServerCapabilities.ClientServerCompatibilityStatus st = + RemoteServerCapabilities.checkClientServerCompatibility( + caps, + remoteOptions, + DigestFunction.Value.SHA256, + ServerCapabilitiesRequirement.EXECUTION); + assertThat(st.isOk()).isTrue(); + } + @Test public void testCheckClientServerCompatibility_cacheCapsOnly() throws Exception { ServerCapabilities caps = diff --git a/third_party/remoteapis/build/bazel/remote/execution/v2/remote_execution.proto b/third_party/remoteapis/build/bazel/remote/execution/v2/remote_execution.proto index 60753f78ab59a5..8e5fb4b81cc163 100644 --- a/third_party/remoteapis/build/bazel/remote/execution/v2/remote_execution.proto +++ b/third_party/remoteapis/build/bazel/remote/execution/v2/remote_execution.proto @@ -104,7 +104,12 @@ service Execution { // send a [PreconditionFailure][google.rpc.PreconditionFailure] error detail // where, for each requested blob not present in the CAS, there is a // `Violation` with a `type` of `MISSING` and a `subject` of - // `"blobs/{hash}/{size}"` indicating the digest of the missing blob. + // `"blobs/{digest_function/}{hash}/{size}"` indicating the digest of the + // missing blob. The `subject` is formatted the same way as the + // `resource_name` provided to + // [ByteStream.Read][google.bytestream.ByteStream.Read], with the leading + // instance name omitted. `digest_function` MUST thus be omitted if its value + // is one of MD5, MURMUR3, SHA1, SHA256, SHA384, SHA512, or VSO. // // The server does not need to guarantee that a call to this method leads to // at most one execution of the action. The server MAY execute the action @@ -204,20 +209,26 @@ service ActionCache { // [Write method][google.bytestream.ByteStream.Write] of the ByteStream API. // // For uncompressed data, The `WriteRequest.resource_name` is of the following form: -// `{instance_name}/uploads/{uuid}/blobs/{hash}/{size}{/optional_metadata}` +// `{instance_name}/uploads/{uuid}/blobs/{digest_function/}{hash}/{size}{/optional_metadata}` // // Where: // * `instance_name` is an identifier, possibly containing multiple path // segments, used to distinguish between the various instances on the server, // in a manner defined by the server. If it is the empty path, the leading // slash is omitted, so that the `resource_name` becomes -// `uploads/{uuid}/blobs/{hash}/{size}{/optional_metadata}`. +// `uploads/{uuid}/blobs/{digest_function/}{hash}/{size}{/optional_metadata}`. // To simplify parsing, a path segment cannot equal any of the following // keywords: `blobs`, `uploads`, `actions`, `actionResults`, `operations`, // `capabilities` or `compressed-blobs`. // * `uuid` is a version 4 UUID generated by the client, used to avoid // collisions between concurrent uploads of the same data. Clients MAY // reuse the same `uuid` for uploading different blobs. +// * `digest_function` is a lowercase string form of a `DigestFunction.Value` +// enum, indicating which digest function was used to compute `hash`. If the +// digest function used is one of MD5, MURMUR3, SHA1, SHA256, SHA384, SHA512, +// or VSO, this component MUST be omitted. In that case the server SHOULD +// infer the digest function using the length of the `hash` and the digest +// functions announced in the server's capabilities. // * `hash` and `size` refer to the [Digest][build.bazel.remote.execution.v2.Digest] // of the data being uploaded. // * `optional_metadata` is implementation specific data, which clients MAY omit. @@ -225,10 +236,11 @@ service ActionCache { // // Data can alternatively be uploaded in compressed form, with the following // `WriteRequest.resource_name` form: -// `{instance_name}/uploads/{uuid}/compressed-blobs/{compressor}/{uncompressed_hash}/{uncompressed_size}{/optional_metadata}` +// `{instance_name}/uploads/{uuid}/compressed-blobs/{compressor}/{digest_function/}{uncompressed_hash}/{uncompressed_size}{/optional_metadata}` // // Where: -// * `instance_name`, `uuid` and `optional_metadata` are defined as above. +// * `instance_name`, `uuid`, `digest_function` and `optional_metadata` are +// defined as above. // * `compressor` is a lowercase string form of a `Compressor.Value` enum // other than `identity`, which is supported by the server and advertised in // [CacheCapabilities.supported_compressor][build.bazel.remote.execution.v2.CacheCapabilities.supported_compressor]. @@ -271,15 +283,17 @@ service ActionCache { // [Read method][google.bytestream.ByteStream.Read] of the ByteStream API. // // For uncompressed data, The `ReadRequest.resource_name` is of the following form: -// `{instance_name}/blobs/{hash}/{size}` -// Where `instance_name`, `hash` and `size` are defined as for uploads. +// `{instance_name}/blobs/{digest_function/}{hash}/{size}` +// Where `instance_name`, `digest_function`, `hash` and `size` are defined as +// for uploads. // // Data can alternatively be downloaded in compressed form, with the following // `ReadRequest.resource_name` form: -// `{instance_name}/compressed-blobs/{compressor}/{uncompressed_hash}/{uncompressed_size}` +// `{instance_name}/compressed-blobs/{compressor}/{digest_function/}{uncompressed_hash}/{uncompressed_size}` // // Where: -// * `instance_name` and `compressor` are defined as for uploads. +// * `instance_name`, `compressor` and `digest_function` are defined as for +// uploads. // * `uncompressed_hash` and `uncompressed_size` refer to the // [Digest][build.bazel.remote.execution.v2.Digest] of the data being // downloaded, once uncompressed. Clients MUST verify that these match @@ -1365,6 +1379,15 @@ message ExecuteRequest { // The server will have a default policy if this is not provided. // This may be applied to both the ActionResult and the associated blobs. ResultsCachePolicy results_cache_policy = 8; + + // The digest function that was used to compute the action digest. + // + // If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, + // SHA384, SHA512, or VSO, the client MAY leave this field unset. In + // that case the server SHOULD infer the digest function using the + // length of the action digest hash and the digest functions announced + // in the server's capabilities. + DigestFunction.Value digest_function = 9; } // A `LogFile` is a log stored in the CAS. @@ -1471,6 +1494,10 @@ message ExecuteOperationMetadata { // [ByteStream.Read][google.bytestream.ByteStream.Read] to stream the // standard error from the endpoint hosting streamed responses. string stderr_stream_name = 4; + + // The client can read this field to view details about the ongoing + // execution. + ExecutedActionMetadata partial_execution_metadata = 5; } // A request message for @@ -1508,6 +1535,15 @@ message GetActionResultRequest { // `output_files` (DEPRECATED since v2.1) in the // [Command][build.bazel.remote.execution.v2.Command] message. repeated string inline_output_files = 5; + + // The digest function that was used to compute the action digest. + // + // If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, + // SHA384, SHA512, or VSO, the client MAY leave this field unset. In + // that case the server SHOULD infer the digest function using the + // length of the action digest hash and the digest functions announced + // in the server's capabilities. + DigestFunction.Value digest_function = 6; } // A request message for @@ -1532,6 +1568,15 @@ message UpdateActionResultRequest { // The server will have a default policy if this is not provided. // This may be applied to both the ActionResult and the associated blobs. ResultsCachePolicy results_cache_policy = 4; + + // The digest function that was used to compute the action digest. + // + // If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, + // SHA384, SHA512, or VSO, the client MAY leave this field unset. In + // that case the server SHOULD infer the digest function using the + // length of the action digest hash and the digest functions announced + // in the server's capabilities. + DigestFunction.Value digest_function = 5; } // A request message for @@ -1544,8 +1589,18 @@ message FindMissingBlobsRequest { // omitted. string instance_name = 1; - // A list of the blobs to check. + // A list of the blobs to check. All digests MUST use the same digest + // function. repeated Digest blob_digests = 2; + + // The digest function of the blobs whose existence is checked. + // + // If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, + // SHA384, SHA512, or VSO, the client MAY leave this field unset. In + // that case the server SHOULD infer the digest function using the + // length of the blob digest hashes and the digest functions announced + // in the server's capabilities. + DigestFunction.Value digest_function = 3; } // A response message for @@ -1560,7 +1615,8 @@ message FindMissingBlobsResponse { message BatchUpdateBlobsRequest { // A request corresponding to a single blob that the client wants to upload. message Request { - // The digest of the blob. This MUST be the digest of `data`. + // The digest of the blob. This MUST be the digest of `data`. All + // digests MUST use the same digest function. Digest digest = 1; // The raw binary data. @@ -1582,6 +1638,16 @@ message BatchUpdateBlobsRequest { // The individual upload requests. repeated Request requests = 2; + + // The digest function that was used to compute the digests of the + // blobs being uploaded. + // + // If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, + // SHA384, SHA512, or VSO, the client MAY leave this field unset. In + // that case the server SHOULD infer the digest function using the + // length of the blob digest hashes and the digest functions announced + // in the server's capabilities. + DigestFunction.Value digest_function = 5; } // A response message for @@ -1610,12 +1676,22 @@ message BatchReadBlobsRequest { // omitted. string instance_name = 1; - // The individual blob digests. + // The individual blob digests. All digests MUST use the same digest + // function. repeated Digest digests = 2; // A list of acceptable encodings for the returned inlined data, in no // particular order. `IDENTITY` is always allowed even if not specified here. repeated Compressor.Value acceptable_compressors = 3; + + // The digest function of the blobs being requested. + // + // If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, + // SHA384, SHA512, or VSO, the client MAY leave this field unset. In + // that case the server SHOULD infer the digest function using the + // length of the blob digest hashes and the digest functions announced + // in the server's capabilities. + DigestFunction.Value digest_function = 4; } // A response message for @@ -1668,6 +1744,16 @@ message GetTreeRequest { // If present, the server will use that token as an offset, returning only // that page and the ones that succeed it. string page_token = 4; + + // The digest function that was used to compute the digest of the root + // directory. + // + // If the digest function used is one of MD5, MURMUR3, SHA1, SHA256, + // SHA384, SHA512, or VSO, the client MAY leave this field unset. In + // that case the server SHOULD infer the digest function using the + // length of the root digest hash and the digest functions announced + // in the server's capabilities. + DigestFunction.Value digest_function = 5; } // A response message for @@ -1743,6 +1829,62 @@ message DigestFunction { // cryptographic hash function and its collision properties are not strongly guaranteed. // See https://github.com/aappleby/smhasher/wiki/MurmurHash3 . MURMUR3 = 7; + + // The SHA-256 digest function, modified to use a Merkle tree for + // large objects. This permits implementations to store large blobs + // as a decomposed sequence of 2^j sized chunks, where j >= 10, + // while being able to validate integrity at the chunk level. + // + // Furthermore, on systems that do not offer dedicated instructions + // for computing SHA-256 hashes (e.g., the Intel SHA and ARMv8 + // cryptographic extensions), SHA256TREE hashes can be computed more + // efficiently than plain SHA-256 hashes by using generic SIMD + // extensions, such as Intel AVX2 or ARM NEON. + // + // SHA256TREE hashes are computed as follows: + // + // - For blobs that are 1024 bytes or smaller, the hash is computed + // using the regular SHA-256 digest function. + // + // - For blobs that are more than 1024 bytes in size, the hash is + // computed as follows: + // + // 1. The blob is partitioned into a left (leading) and right + // (trailing) blob. These blobs have lengths m and n + // respectively, where m = 2^k and 0 < n <= m. + // + // 2. Hashes of the left and right blob, Hash(left) and + // Hash(right) respectively, are computed by recursively + // applying the SHA256TREE algorithm. + // + // 3. A single invocation is made to the SHA-256 block cipher with + // the following parameters: + // + // M = Hash(left) || Hash(right) + // H = { + // 0xcbbb9d5d, 0x629a292a, 0x9159015a, 0x152fecd8, + // 0x67332667, 0x8eb44a87, 0xdb0c2e0d, 0x47b5481d, + // } + // + // The values of H are the leading fractional parts of the + // square roots of the 9th to the 16th prime number (23 to 53). + // This differs from plain SHA-256, where the first eight prime + // numbers (2 to 19) are used, thereby preventing trivial hash + // collisions between small and large objects. + // + // 4. The hash of the full blob can then be obtained by + // concatenating the outputs of the block cipher: + // + // Hash(blob) = a || b || c || d || e || f || g || h + // + // Addition of the original values of H, as normally done + // through the use of the Davies-Meyer structure, is not + // performed. This isn't necessary, as the block cipher is only + // invoked once. + // + // Test vectors of this digest function can be found in the + // accompanying sha256tree_test_vectors.txt file. + SHA256TREE = 8; } } @@ -1803,6 +1945,9 @@ message Compressor { // It is advised to use algorithms such as Zstandard instead, as // those are faster and/or provide a better compression ratio. DEFLATE = 2; + + // Brotli compression. + BROTLI = 3; } } @@ -1843,7 +1988,10 @@ message CacheCapabilities { // Capabilities of the remote execution system. message ExecutionCapabilities { - // Remote execution may only support a single digest function. + // Legacy field for indicating which digest function is supported by the + // remote execution system. It MUST be set to a value other than UNKNOWN. + // Implementations should consider the repeated digest_functions field + // first, falling back to this singular field if digest_functions is unset. DigestFunction.Value digest_function = 1; // Whether remote execution is enabled for the particular server/instance. @@ -1854,6 +2002,20 @@ message ExecutionCapabilities { // Supported node properties. repeated string supported_node_properties = 4; + + // All the digest functions supported by the remote execution system. + // If this field is set, it MUST also contain digest_function. + // + // Even if the remote execution system announces support for multiple + // digest functions, individual execution requests may only reference + // CAS objects using a single digest function. For example, it is not + // permitted to execute actions having both MD5 and SHA-256 hashed + // files in their input root. + // + // The CAS objects referenced by action results generated by the + // remote execution system MUST use the same digest function as the + // one used to construct the action. + repeated DigestFunction.Value digest_functions = 5; } // Details for the tool used to call the API. From bade2e905d42ddc81851ba6e65967a1d3b8db76d Mon Sep 17 00:00:00 2001 From: George Gensure Date: Tue, 25 Jul 2023 05:22:23 -0400 Subject: [PATCH 002/125] Release 6.4.0 remote (#18959) * Include stack trace in all gRPC errors when --verbose_failures is set. Also refactor a couple places where the stack trace was included in an ad-hoc manner, and force Utils.grpcAwareErrorMessage callers to be explicit to avoid future instances. Closes #16086. PiperOrigin-RevId: 502854490 Change-Id: Id2d6a1728630fffea9399b406378c7f173b247bd * Avoid discarding SRE state for IO cause Unwrapping all StatusRuntimeExceptions in in ReferenceCountedChannel when caused by IOException will discard critical tracing and retriability. The Retrier evaluations may not see an SRE in the causal chain, and presume it is invariably an unretriable exception. In general, IOExceptions as SRE wrappers are unsuitable containers and are routinely misued either for identification (grpc aware status), or capture (handleInitError). Partially addresses #18764 (retries will occur with SSL handshake timeout, but the actual connection will not be retried) Closes #18836. PiperOrigin-RevId: 546037698 Change-Id: I7f6efcb857c557aa97ad3df085fc032c8538eb9a * Operation stream termination is not an error According to the GrpcRemoteExecutor when it occurs after a !done operation response. Remove the error from the ExperimentalRemoteGrpcExecutor and reinforce both with tests. Update the FakeExecutionService to generate compatible error responses that appear in the ExecuteResponse, rather than the operation error field, per the REAPI spec. Made required adjustments to ExGRE Test invocations to avoid the ExecutionStatusException interpretation of DEADLINE_EXCEEDED -> FAILED_PRECONDITION in ExecuteResponse. Closes #18785. PiperOrigin-RevId: 546925894 Change-Id: I7a489c8bc936a83cfd94e0138437f3fe6d152da8 * Done operations must be reexecuted Any operation with done == true as reported by the server is not expected to change its result on subsequent waitExecution calls. To properly retry, this action must be reexecuted, if it was truly transient, to achieve a definitive result. Submit a transient status for retry, disallow special behaviors for NOT_FOUND as covered by done observation, and consider method type when handling operation streams. Closes #18943. PiperOrigin-RevId: 548680656 Change-Id: Ib2c9887ead1fbd3de97761db6e8b4077783ad03c --------- Co-authored-by: Tiago Quelhas --- .../ExperimentalGrpcRemoteExecutor.java | 60 ++-- .../lib/remote/ReferenceCountedChannel.java | 49 ++- .../build/lib/remote/RemoteModule.java | 24 +- .../lib/remote/RemoteServerCapabilities.java | 27 +- .../build/lib/remote/RemoteSpawnCache.java | 9 +- .../build/lib/remote/RemoteSpawnRunner.java | 7 +- .../devtools/build/lib/remote/util/Utils.java | 2 +- .../ExperimentalGrpcRemoteExecutorTest.java | 308 ++-------------- .../lib/remote/FakeExecutionService.java | 17 +- .../lib/remote/GrpcRemoteExecutorTest.java | 103 ++++++ .../remote/GrpcRemoteExecutorTestBase.java | 330 ++++++++++++++++++ .../devtools/build/lib/remote/UtilsTest.java | 27 +- 12 files changed, 601 insertions(+), 362 deletions(-) create mode 100644 src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutorTest.java create mode 100644 src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutorTestBase.java diff --git a/src/main/java/com/google/devtools/build/lib/remote/ExperimentalGrpcRemoteExecutor.java b/src/main/java/com/google/devtools/build/lib/remote/ExperimentalGrpcRemoteExecutor.java index 428da883fba69d..003115278b3fd0 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/ExperimentalGrpcRemoteExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/remote/ExperimentalGrpcRemoteExecutor.java @@ -146,10 +146,12 @@ ExecuteResponse start() throws IOException, InterruptedException { // retrying when received a unauthenticated error, and propagate to refreshIfUnauthenticated // which will then call retrier again. It will reset the retry time counter so we could // retry more than --remote_retry times which is not expected. - response = - retrier.execute( - () -> Utils.refreshIfUnauthenticated(this::execute, callCredentialsProvider), - executeBackoff); + if (lastOperation == null) { + response = + retrier.execute( + () -> Utils.refreshIfUnauthenticated(this::execute, callCredentialsProvider), + executeBackoff); + } // If no response from Execute(), use WaitExecution() in a "loop" which is implemented // inside the retry block. @@ -177,7 +179,7 @@ ExecuteResponse execute() throws IOException { try { Iterator operationStream = executeFunction.apply(request); - return handleOperationStream(operationStream); + return handleOperationStream(operationStream, /* waitExecution= */ false); } catch (Throwable e) { // If lastOperation is not null, we know the execution request is accepted by the server. In // this case, we will fallback to WaitExecution() loop when the stream is broken. @@ -197,33 +199,43 @@ ExecuteResponse waitExecution() throws IOException { WaitExecutionRequest.newBuilder().setName(lastOperation.getName()).build(); try { Iterator operationStream = waitExecutionFunction.apply(request); - return handleOperationStream(operationStream); + return handleOperationStream(operationStream, /* waitExecution= */ true); + } catch (StatusRuntimeException e) { + throw new IOException(e); } catch (Throwable e) { - // A NOT_FOUND error means Operation was lost on the server, retry Execute(). - // - // However, we only retry Execute() if executeBackoff should retry. Also increase the retry - // counter at the same time (done by nextDelayMillis()). - if (e instanceof StatusRuntimeException) { - StatusRuntimeException sre = (StatusRuntimeException) e; - if (sre.getStatus().getCode() == Code.NOT_FOUND - && executeBackoff.nextDelayMillis(sre) >= 0) { - lastOperation = null; - return null; - } - } + lastOperation = null; throw new IOException(e); } } /** Process a stream of operations from Execute() or WaitExecution(). */ - ExecuteResponse handleOperationStream(Iterator operationStream) throws IOException { + @Nullable + ExecuteResponse handleOperationStream( + Iterator operationStream, boolean waitExecution) throws IOException { try { while (operationStream.hasNext()) { Operation operation = operationStream.next(); - ExecuteResponse response = extractResponseOrThrowIfError(operation); - // At this point, we successfully received a response that is not an error. - lastOperation = operation; + // Either done or should be repeated + lastOperation = operation.getDone() ? null : operation; + + ExecuteResponse response; + try { + response = extractResponseOrThrowIfError(operation); + } catch (StatusRuntimeException e) { + // An operation error means Operation has been terminally completed, retry Execute(). + // + // However, we only retry Execute() if executeBackoff should retry. Also increase the + // retry + // counter at the same time (done by nextDelayMillis()). + if (waitExecution + && (retrier.isRetriable(e) || e.getStatus().getCode() == Code.NOT_FOUND) + && executeBackoff.nextDelayMillis(e) >= 0) { + lastOperation = null; + return null; + } + throw e; + } // We don't want to reset executeBackoff since if there is an error: // 1. If happened before we received a first response, we want to ensure the retry @@ -258,8 +270,8 @@ ExecuteResponse handleOperationStream(Iterator operationStream) throw } } - // The operation completed successfully but without a result. - throw new IOException("Remote server error: execution terminated with no result"); + // The operation stream completed successfully but without a result. + return null; } finally { close(operationStream); } diff --git a/src/main/java/com/google/devtools/build/lib/remote/ReferenceCountedChannel.java b/src/main/java/com/google/devtools/build/lib/remote/ReferenceCountedChannel.java index 8e4e95afd56810..bc2f6ec59940d9 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/ReferenceCountedChannel.java +++ b/src/main/java/com/google/devtools/build/lib/remote/ReferenceCountedChannel.java @@ -13,10 +13,11 @@ // limitations under the License. package com.google.devtools.build.lib.remote; -import static com.google.common.base.Throwables.throwIfInstanceOf; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; +import com.google.common.base.Throwables; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; import com.google.devtools.build.lib.remote.grpc.ChannelConnectionFactory; import com.google.devtools.build.lib.remote.grpc.ChannelConnectionFactory.ChannelConnection; import com.google.devtools.build.lib.remote.grpc.DynamicConnectionPool; @@ -28,9 +29,12 @@ import io.netty.util.ReferenceCounted; import io.reactivex.rxjava3.annotations.CheckReturnValue; import io.reactivex.rxjava3.core.Single; +import io.reactivex.rxjava3.core.SingleObserver; import io.reactivex.rxjava3.core.SingleSource; +import io.reactivex.rxjava3.disposables.Disposable; import io.reactivex.rxjava3.functions.Function; import java.io.IOException; +import java.util.concurrent.ExecutionException; /** * A wrapper around a {@link DynamicConnectionPool} exposing {@link Channel} and a reference count. @@ -80,19 +84,48 @@ public ListenableFuture withChannelFuture( } public T withChannelBlocking(Function source) - throws IOException, InterruptedException { + throws ExecutionException, IOException, InterruptedException { try { - return withChannel(channel -> Single.just(source.apply(channel))).blockingGet(); - } catch (RuntimeException e) { + return withChannelBlockingGet(source); + } catch (ExecutionException e) { Throwable cause = e.getCause(); - if (cause != null) { - throwIfInstanceOf(cause, IOException.class); - throwIfInstanceOf(cause, InterruptedException.class); - } + Throwables.throwIfInstanceOf(cause, IOException.class); + Throwables.throwIfUnchecked(cause); throw e; } } + // prevents rxjava silent possible wrap of RuntimeException and misinterpretation + private T withChannelBlockingGet(Function source) + throws ExecutionException, InterruptedException { + SettableFuture future = SettableFuture.create(); + withChannel(channel -> Single.just(source.apply(channel))) + .subscribe( + new SingleObserver() { + @Override + public void onError(Throwable t) { + future.setException(t); + } + + @Override + public void onSuccess(T t) { + future.set(t); + } + + @Override + public void onSubscribe(Disposable d) { + future.addListener( + () -> { + if (future.isCancelled()) { + d.dispose(); + } + }, + directExecutor()); + } + }); + return future.get(); + } + @CheckReturnValue public Single withChannel(Function> source) { return dynamicConnectionPool diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java index 2795b133a1224f..18db228f3e254b 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java @@ -188,6 +188,7 @@ private static boolean shouldEnableRemoteDownloader(RemoteOptions options) { return !Strings.isNullOrEmpty(options.remoteDownloader); } + @Nullable private static ServerCapabilities getAndVerifyServerCapabilities( RemoteOptions remoteOptions, ReferenceCountedChannel channel, @@ -535,9 +536,12 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { digestUtil, ServerCapabilitiesRequirement.CACHE); } - } catch (IOException e) { + } catch (AbruptExitException e) { + throw e; // prevent abrupt interception + } catch (Exception e) { String errorMessage = - "Failed to query remote execution capabilities: " + Utils.grpcAwareErrorMessage(e); + "Failed to query remote execution capabilities: " + + Utils.grpcAwareErrorMessage(e, verboseFailures); if (remoteOptions.remoteLocalFallback) { if (verboseFailures) { errorMessage += System.lineSeparator() + Throwables.getStackTraceAsString(e); @@ -559,12 +563,12 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { if (Strings.isNullOrEmpty(remoteBytestreamUriPrefix)) { try { remoteBytestreamUriPrefix = cacheChannel.withChannelBlocking(Channel::authority); - } catch (IOException e) { + } catch (Exception e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } handleInitFailure(env, e, Code.CACHE_INIT_FAILURE); return; - } catch (InterruptedException e) { - handleInitFailure(env, new IOException(e), Code.CACHE_INIT_FAILURE); - return; } if (!Strings.isNullOrEmpty(remoteOptions.remoteInstanceName)) { remoteBytestreamUriPrefix += "/" + remoteOptions.remoteInstanceName; @@ -587,7 +591,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { !remoteOptions.remoteOutputsMode.downloadAllOutputs(), digestUtil, cacheClient); - } catch (IOException e) { + } catch (Exception e) { handleInitFailure(env, e, Code.CACHE_INIT_FAILURE); return; } @@ -652,7 +656,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { !remoteOptions.remoteOutputsMode.downloadAllOutputs(), digestUtil, cacheClient); - } catch (IOException e) { + } catch (Exception e) { handleInitFailure(env, e, Code.CACHE_INIT_FAILURE); return; } @@ -698,7 +702,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException { } private static void handleInitFailure( - CommandEnvironment env, IOException e, Code remoteExecutionCode) { + CommandEnvironment env, Exception e, Code remoteExecutionCode) { env.getReporter().handle(Event.error(e.getMessage())); env.getBlazeModuleEnvironment() .exit( @@ -793,7 +797,7 @@ private static void checkClientServerCompatibility( } @Override - public void afterCommand() throws AbruptExitException { + public void afterCommand() { Preconditions.checkNotNull(blockWaitingModule, "blockWaitingModule must not be null"); // Some cleanup tasks must wait until every other BlazeModule's afterCommand() has run, as diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteServerCapabilities.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteServerCapabilities.java index a69d4f60e1faea..e13a973ba9bd54 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteServerCapabilities.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteServerCapabilities.java @@ -31,7 +31,6 @@ import com.google.devtools.build.lib.remote.util.TracingMetadataUtils; import io.grpc.CallCredentials; import io.grpc.Channel; -import io.grpc.StatusRuntimeException; import java.io.IOException; import java.util.List; import java.util.concurrent.TimeUnit; @@ -73,21 +72,17 @@ public ServerCapabilities get(String buildRequestId, String commandId) RequestMetadata metadata = TracingMetadataUtils.buildMetadata(buildRequestId, commandId, "capabilities", null); RemoteActionExecutionContext context = RemoteActionExecutionContext.create(metadata); - try { - GetCapabilitiesRequest request = - instanceName == null - ? GetCapabilitiesRequest.getDefaultInstance() - : GetCapabilitiesRequest.newBuilder().setInstanceName(instanceName).build(); - return retrier.execute( - () -> - channel.withChannelBlocking( - channel -> capabilitiesBlockingStub(context, channel).getCapabilities(request))); - } catch (StatusRuntimeException e) { - if (e.getCause() instanceof IOException) { - throw (IOException) e.getCause(); - } - throw new IOException(e); - } + GetCapabilitiesRequest request = + instanceName == null + ? GetCapabilitiesRequest.getDefaultInstance() + : GetCapabilitiesRequest.newBuilder().setInstanceName(instanceName).build(); + ServerCapabilities caps = + retrier.execute( + () -> + channel.withChannelBlocking( + channel -> + capabilitiesBlockingStub(context, channel).getCapabilities(request))); + return caps; } static class ClientServerCompatibilityStatus { diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java index fd4205bf2925c0..6207d11edce643 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java @@ -19,7 +19,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Stopwatch; -import com.google.common.base.Throwables; import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.FileArtifactValue; @@ -141,13 +140,7 @@ public CacheHandle lookup(Spawn spawn, SpawnExecutionContext context) if (BulkTransferException.allCausedByCacheNotFoundException(e)) { // Intentionally left blank } else { - String errorMessage; - if (!verboseFailures) { - errorMessage = Utils.grpcAwareErrorMessage(e); - } else { - // On --verbose_failures print the whole stack trace - errorMessage = "\n" + Throwables.getStackTraceAsString(e); - } + String errorMessage = Utils.grpcAwareErrorMessage(e, verboseFailures); if (isNullOrEmpty(errorMessage)) { errorMessage = e.getClass().getSimpleName(); } diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java index ec7e7d7e86d6e2..4be0774599b770 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java @@ -30,7 +30,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Stopwatch; -import com.google.common.base.Throwables; import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.devtools.build.lib.actions.ActionInput; import com.google.devtools.build.lib.actions.CommandLines.ParamFileActionInput; @@ -573,11 +572,7 @@ private SpawnResult handleError( catastrophe = false; } - String errorMessage = Utils.grpcAwareErrorMessage(exception); - if (verboseFailures) { - // On --verbose_failures print the whole stack trace - errorMessage += "\n" + Throwables.getStackTraceAsString(exception); - } + String errorMessage = Utils.grpcAwareErrorMessage(exception, verboseFailures); if (exception.getCause() instanceof ExecutionStatusException) { ExecutionStatusException e = (ExecutionStatusException) exception.getCause(); diff --git a/src/main/java/com/google/devtools/build/lib/remote/util/Utils.java b/src/main/java/com/google/devtools/build/lib/remote/util/Utils.java index 720560b8e925af..4ca78d304adb7a 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/util/Utils.java +++ b/src/main/java/com/google/devtools/build/lib/remote/util/Utils.java @@ -350,7 +350,7 @@ private static String executionStatusExceptionErrorMessage(ExecutionStatusExcept + errorDetailsMessage(status.getDetailsList()); } - public static String grpcAwareErrorMessage(IOException e) { + private static String grpcAwareErrorMessage(IOException e) { io.grpc.Status errStatus = io.grpc.Status.fromThrowable(e); if (e.getCause() instanceof ExecutionStatusException) { // Display error message returned by the remote service. diff --git a/src/test/java/com/google/devtools/build/lib/remote/ExperimentalGrpcRemoteExecutorTest.java b/src/test/java/com/google/devtools/build/lib/remote/ExperimentalGrpcRemoteExecutorTest.java index d42021a4027141..bbfcd8a3a7e5b8 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/ExperimentalGrpcRemoteExecutorTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/ExperimentalGrpcRemoteExecutorTest.java @@ -17,258 +17,63 @@ import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.Assert.assertThrows; -import build.bazel.remote.execution.v2.ActionResult; -import build.bazel.remote.execution.v2.Digest; -import build.bazel.remote.execution.v2.ExecuteRequest; import build.bazel.remote.execution.v2.ExecuteResponse; -import build.bazel.remote.execution.v2.ExecutionCapabilities; -import build.bazel.remote.execution.v2.OutputFile; -import build.bazel.remote.execution.v2.RequestMetadata; import build.bazel.remote.execution.v2.ServerCapabilities; import com.google.common.util.concurrent.ListeningScheduledExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.devtools.build.lib.authandtls.CallCredentialsProvider; import com.google.devtools.build.lib.remote.RemoteRetrier.ExponentialBackoff; import com.google.devtools.build.lib.remote.common.OperationObserver; -import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext; -import com.google.devtools.build.lib.remote.grpc.ChannelConnectionFactory; -import com.google.devtools.build.lib.remote.options.RemoteOptions; +import com.google.devtools.build.lib.remote.common.RemoteExecutionClient; import com.google.devtools.build.lib.remote.util.TestUtils; -import com.google.devtools.build.lib.remote.util.TracingMetadataUtils; -import com.google.devtools.common.options.Options; -import com.google.longrunning.Operation; -import com.google.rpc.Code; -import io.grpc.ManagedChannel; -import io.grpc.Server; import io.grpc.Status; -import io.grpc.inprocess.InProcessChannelBuilder; -import io.grpc.inprocess.InProcessServerBuilder; -import io.reactivex.rxjava3.core.Single; import java.io.IOException; -import java.util.ArrayList; -import java.util.List; import java.util.concurrent.Executors; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; /** Tests for {@link ExperimentalGrpcRemoteExecutor}. */ @RunWith(JUnit4.class) -public class ExperimentalGrpcRemoteExecutorTest { - - private RemoteActionExecutionContext context; - private FakeExecutionService executionService; - private RemoteOptions remoteOptions; - private Server fakeServer; +public class ExperimentalGrpcRemoteExecutorTest extends GrpcRemoteExecutorTestBase { private ListeningScheduledExecutorService retryService; - ExperimentalGrpcRemoteExecutor executor; - - private static final int MAX_RETRY_ATTEMPTS = 5; - - private static final OutputFile DUMMY_OUTPUT = - OutputFile.newBuilder() - .setPath("dummy.txt") - .setDigest( - Digest.newBuilder() - .setHash("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") - .setSizeBytes(0) - .build()) - .build(); - - private static final ExecuteRequest DUMMY_REQUEST = - ExecuteRequest.newBuilder() - .setInstanceName("dummy") - .setActionDigest( - Digest.newBuilder() - .setHash("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") - .setSizeBytes(0) - .build()) - .build(); - - private static final ExecuteResponse DUMMY_RESPONSE = - ExecuteResponse.newBuilder() - .setResult(ActionResult.newBuilder().addOutputFiles(DUMMY_OUTPUT).build()) - .build(); - - @Before - public final void setUp() throws Exception { - context = RemoteActionExecutionContext.create(RequestMetadata.getDefaultInstance()); - - executionService = new FakeExecutionService(); - - String fakeServerName = "fake server for " + getClass(); - // Use a mutable service registry for later registering the service impl for each test case. - fakeServer = - InProcessServerBuilder.forName(fakeServerName) - .addService(executionService) - .directExecutor() - .build() - .start(); - remoteOptions = Options.getDefaults(RemoteOptions.class); - remoteOptions.remoteMaxRetryAttempts = MAX_RETRY_ATTEMPTS; - - retryService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1)); + @Override + protected RemoteExecutionClient createExecutionService( + ServerCapabilities caps, ReferenceCountedChannel channel) throws Exception { RemoteRetrier retrier = TestUtils.newRemoteRetrier( () -> new ExponentialBackoff(remoteOptions), RemoteRetrier.RETRIABLE_GRPC_ERRORS, retryService); - ReferenceCountedChannel channel = - new ReferenceCountedChannel( - new ChannelConnectionFactory() { - @Override - public Single create() { - ManagedChannel ch = - InProcessChannelBuilder.forName(fakeServerName) - .intercept(TracingMetadataUtils.newExecHeadersInterceptor(remoteOptions)) - .directExecutor() - .build(); - return Single.just(new ChannelConnection(ch)); - } - - @Override - public int maxConcurrency() { - return 100; - } - }); - ServerCapabilities caps = - ServerCapabilities.newBuilder() - .setExecutionCapabilities( - ExecutionCapabilities.newBuilder().setExecEnabled(true).build()) - .build(); + return new ExperimentalGrpcRemoteExecutor( + caps, remoteOptions, channel, CallCredentialsProvider.NO_CREDENTIALS, retrier); + } - executor = - new ExperimentalGrpcRemoteExecutor( - caps, remoteOptions, channel, CallCredentialsProvider.NO_CREDENTIALS, retrier); + @Override + public void setUp() throws Exception { + retryService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1)); + super.setUp(); } - @After + @Override public void tearDown() throws Exception { retryService.shutdownNow(); retryService.awaitTermination( com.google.devtools.build.lib.testutil.TestUtils.WAIT_TIMEOUT_SECONDS, SECONDS); - - fakeServer.shutdownNow(); - fakeServer.awaitTermination(); - - executor.close(); - } - - @Test - public void executeRemotely_smoke() throws Exception { - executionService.whenExecute(DUMMY_REQUEST).thenAck().thenAck().thenDone(DUMMY_RESPONSE); - - ExecuteResponse response = - executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); - - assertThat(response).isEqualTo(DUMMY_RESPONSE); - assertThat(executionService.getExecTimes()).isEqualTo(1); - } - - @Test - public void executeRemotely_errorInOperation_retryExecute() throws Exception { - executionService.whenExecute(DUMMY_REQUEST).thenError(new RuntimeException("Unavailable")); - executionService.whenExecute(DUMMY_REQUEST).thenError(Code.UNAVAILABLE); - executionService.whenExecute(DUMMY_REQUEST).thenAck().thenDone(DUMMY_RESPONSE); - - ExecuteResponse response = - executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); - - assertThat(executionService.getExecTimes()).isEqualTo(3); - assertThat(response).isEqualTo(DUMMY_RESPONSE); - } - - @Test - public void executeRemotely_errorInResponse_retryExecute() throws Exception { - executionService - .whenExecute(DUMMY_REQUEST) - .thenDone( - ExecuteResponse.newBuilder() - .setStatus(com.google.rpc.Status.newBuilder().setCode(Code.UNAVAILABLE_VALUE)) - .build()); - executionService.whenExecute(DUMMY_REQUEST).thenAck().thenDone(DUMMY_RESPONSE); - - ExecuteResponse response = - executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); - - assertThat(executionService.getExecTimes()).isEqualTo(2); - assertThat(response).isEqualTo(DUMMY_RESPONSE); - } - - @Test - public void executeRemotely_unretriableErrorInResponse_reportError() { - executionService - .whenExecute(DUMMY_REQUEST) - .thenDone( - ExecuteResponse.newBuilder() - .setStatus(com.google.rpc.Status.newBuilder().setCode(Code.INVALID_ARGUMENT_VALUE)) - .build()); - executionService.whenExecute(DUMMY_REQUEST).thenAck().thenDone(DUMMY_RESPONSE); - - IOException e = - assertThrows( - IOException.class, - () -> { - executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); - }); - - assertThat(e).hasMessageThat().contains("INVALID_ARGUMENT"); - assertThat(executionService.getExecTimes()).isEqualTo(1); - } - - @Test - public void executeRemotely_retryExecuteAndFail() { - for (int i = 0; i <= MAX_RETRY_ATTEMPTS * 2; ++i) { - executionService.whenExecute(DUMMY_REQUEST).thenError(Code.UNAVAILABLE); - } - - IOException exception = - assertThrows( - IOException.class, - () -> { - executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); - }); - - assertThat(executionService.getExecTimes()).isEqualTo(MAX_RETRY_ATTEMPTS + 1); - assertThat(exception).hasMessageThat().contains("UNAVAILABLE"); - } - - @Test - public void executeRemotely_executeAndWait() throws Exception { - executionService.whenExecute(DUMMY_REQUEST).thenAck().thenError(Code.UNAVAILABLE); - executionService.whenWaitExecution(DUMMY_REQUEST).thenDone(DUMMY_RESPONSE); - - ExecuteResponse response = - executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); - - assertThat(executionService.getExecTimes()).isEqualTo(1); - assertThat(executionService.getWaitTimes()).isEqualTo(1); - assertThat(response).isEqualTo(DUMMY_RESPONSE); - } - - @Test - public void executeRemotely_executeAndRetryWait() throws Exception { - executionService.whenExecute(DUMMY_REQUEST).thenAck().thenError(Code.UNAVAILABLE); - executionService.whenWaitExecution(DUMMY_REQUEST).thenDone(DUMMY_RESPONSE); - - ExecuteResponse response = - executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); - - assertThat(executionService.getExecTimes()).isEqualTo(1); - assertThat(executionService.getWaitTimes()).isEqualTo(1); - assertThat(response).isEqualTo(DUMMY_RESPONSE); + super.tearDown(); } @Test public void executeRemotely_executeAndRetryWait_forever() throws Exception { - executionService.whenExecute(DUMMY_REQUEST).thenAck().thenError(Code.UNAVAILABLE); + executionService.whenExecute(DUMMY_REQUEST).thenAck().finish(); int errorTimes = MAX_RETRY_ATTEMPTS * 2; for (int i = 0; i < errorTimes; ++i) { - executionService.whenWaitExecution(DUMMY_REQUEST).thenAck().thenError(Code.DEADLINE_EXCEEDED); + executionService + .whenWaitExecution(DUMMY_REQUEST) + .thenAck() + .thenError(Status.DEADLINE_EXCEEDED.asRuntimeException()); } executionService.whenWaitExecution(DUMMY_REQUEST).thenDone(DUMMY_RESPONSE); @@ -282,9 +87,11 @@ public void executeRemotely_executeAndRetryWait_forever() throws Exception { @Test public void executeRemotely_executeAndRetryWait_failForConsecutiveErrors() { - executionService.whenExecute(DUMMY_REQUEST).thenAck().thenError(Code.UNAVAILABLE); + executionService.whenExecute(DUMMY_REQUEST).thenAck().finish(); for (int i = 0; i < MAX_RETRY_ATTEMPTS * 2; ++i) { - executionService.whenWaitExecution(DUMMY_REQUEST).thenError(Code.UNAVAILABLE); + executionService + .whenWaitExecution(DUMMY_REQUEST) + .thenError(Status.UNAVAILABLE.asRuntimeException()); } assertThrows( @@ -340,24 +147,14 @@ public void executeRemotely_responseWithoutResult_shouldNotCrash() { assertThat(executionService.getExecTimes()).isEqualTo(1); } - @Test - public void executeRemotely_retryExecuteWhenUnauthenticated() - throws IOException, InterruptedException { - executionService.whenExecute(DUMMY_REQUEST).thenError(Code.UNAUTHENTICATED); - executionService.whenExecute(DUMMY_REQUEST).thenAck().thenDone(DUMMY_RESPONSE); - - ExecuteResponse response = - executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); - - assertThat(executionService.getExecTimes()).isEqualTo(2); - assertThat(response).isEqualTo(DUMMY_RESPONSE); - } - @Test public void executeRemotely_retryWaitExecutionWhenUnauthenticated() throws IOException, InterruptedException { - executionService.whenExecute(DUMMY_REQUEST).thenAck().thenError(Code.UNAVAILABLE); - executionService.whenWaitExecution(DUMMY_REQUEST).thenAck().thenError(Code.UNAUTHENTICATED); + executionService.whenExecute(DUMMY_REQUEST).thenAck().finish(); + executionService + .whenWaitExecution(DUMMY_REQUEST) + .thenAck() + .thenError(Status.UNAUTHENTICATED.asRuntimeException()); executionService.whenWaitExecution(DUMMY_REQUEST).thenAck().thenDone(DUMMY_RESPONSE); ExecuteResponse response = @@ -367,53 +164,4 @@ public void executeRemotely_retryWaitExecutionWhenUnauthenticated() assertThat(executionService.getWaitTimes()).isEqualTo(2); assertThat(response).isEqualTo(DUMMY_RESPONSE); } - - @Test - public void executeRemotely_retryExecuteIfNotFound() throws IOException, InterruptedException { - executionService.whenExecute(DUMMY_REQUEST).thenAck().thenError(Code.UNAVAILABLE); - executionService.whenWaitExecution(DUMMY_REQUEST).thenError(Code.NOT_FOUND); - executionService.whenExecute(DUMMY_REQUEST).thenAck().thenError(Code.UNAVAILABLE); - executionService.whenWaitExecution(DUMMY_REQUEST).thenDone(DUMMY_RESPONSE); - - ExecuteResponse response = - executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); - - assertThat(executionService.getExecTimes()).isEqualTo(2); - assertThat(executionService.getWaitTimes()).isEqualTo(2); - assertThat(response).isEqualTo(DUMMY_RESPONSE); - } - - @Test - public void executeRemotely_notFoundLoop_reportError() { - for (int i = 0; i <= MAX_RETRY_ATTEMPTS * 2; ++i) { - executionService.whenExecute(DUMMY_REQUEST).thenAck().thenError(Code.UNAVAILABLE); - executionService.whenWaitExecution(DUMMY_REQUEST).thenAck().thenError(Code.NOT_FOUND); - } - - IOException e = - assertThrows( - IOException.class, - () -> { - executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); - }); - - assertThat(e).hasCauseThat().isInstanceOf(ExecutionStatusException.class); - ExecutionStatusException executionStatusException = (ExecutionStatusException) e.getCause(); - assertThat(executionStatusException.getStatus().getCode()).isEqualTo(Status.Code.NOT_FOUND); - assertThat(executionService.getExecTimes()).isEqualTo(MAX_RETRY_ATTEMPTS + 1); - assertThat(executionService.getWaitTimes()).isEqualTo(MAX_RETRY_ATTEMPTS + 1); - } - - @Test - public void executeRemotely_notifyObserver() throws IOException, InterruptedException { - executionService.whenExecute(DUMMY_REQUEST).thenAck().thenDone(DUMMY_RESPONSE); - - List notified = new ArrayList<>(); - executor.executeRemotely(context, DUMMY_REQUEST, notified::add); - - assertThat(notified) - .containsExactly( - FakeExecutionService.ackOperation(DUMMY_REQUEST), - FakeExecutionService.doneOperation(DUMMY_REQUEST, DUMMY_RESPONSE)); - } } diff --git a/src/test/java/com/google/devtools/build/lib/remote/FakeExecutionService.java b/src/test/java/com/google/devtools/build/lib/remote/FakeExecutionService.java index 4c9d93989449dc..59170db624241b 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/FakeExecutionService.java +++ b/src/test/java/com/google/devtools/build/lib/remote/FakeExecutionService.java @@ -115,12 +115,17 @@ public void thenDone(ExecuteResponse response) { } public void thenError(Code code) { + // From REAPI Spec: + // > Errors discovered during creation of the `Operation` will be reported + // > as gRPC Status errors, while errors that occurred while running the + // > action will be reported in the `status` field of the `ExecuteResponse`. The + // > server MUST NOT set the `error` field of the `Operation` proto. Operation operation = - Operation.newBuilder() - .setName(getResourceName(request)) - .setDone(true) - .setError(Status.newBuilder().setCode(code.getNumber())) - .build(); + doneOperation( + request, + ExecuteResponse.newBuilder() + .setStatus(Status.newBuilder().setCode(code.getNumber())) + .build()); operations.add(() -> operation); finish(); } @@ -133,7 +138,7 @@ public void thenError(RuntimeException e) { finish(); } - private void finish() { + public void finish() { String name = getResourceName(request); provider.append(name, ImmutableList.copyOf(operations)); } diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutorTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutorTest.java new file mode 100644 index 00000000000000..b5af2b64a0a5c7 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutorTest.java @@ -0,0 +1,103 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.remote; + +import static com.google.common.truth.Truth.assertThat; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.Assert.assertThrows; + +import build.bazel.remote.execution.v2.ExecuteResponse; +import build.bazel.remote.execution.v2.ServerCapabilities; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.devtools.build.lib.authandtls.CallCredentialsProvider; +import com.google.devtools.build.lib.remote.RemoteRetrier.ExponentialBackoff; +import com.google.devtools.build.lib.remote.common.OperationObserver; +import com.google.devtools.build.lib.remote.common.RemoteExecutionClient; +import com.google.devtools.build.lib.remote.util.TestUtils; +import com.google.rpc.Code; +import java.io.IOException; +import java.util.concurrent.Executors; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link GrpcRemoteExecutor}. */ +@RunWith(JUnit4.class) +public class GrpcRemoteExecutorTest extends GrpcRemoteExecutorTestBase { + private ListeningScheduledExecutorService retryService; + + @Override + public void setUp() throws Exception { + retryService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1)); + super.setUp(); + } + + @Override + protected RemoteExecutionClient createExecutionService( + ServerCapabilities caps, ReferenceCountedChannel channel) throws Exception { + RemoteRetrier retrier = + TestUtils.newRemoteRetrier( + () -> new ExponentialBackoff(remoteOptions), + RemoteRetrier.RETRIABLE_GRPC_EXEC_ERRORS, + retryService); + + return new GrpcRemoteExecutor(caps, channel, CallCredentialsProvider.NO_CREDENTIALS, retrier); + } + + @Override + public void tearDown() throws Exception { + retryService.shutdownNow(); + retryService.awaitTermination( + com.google.devtools.build.lib.testutil.TestUtils.WAIT_TIMEOUT_SECONDS, SECONDS); + super.tearDown(); + } + + @Test + public void executeRemotely_operationWithoutResult_crashes() { + executionService.whenExecute(DUMMY_REQUEST).thenDone(); + + assertThrows( + IllegalStateException.class, + () -> executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP)); + // Shouldn't retry in this case + assertThat(executionService.getExecTimes()).isEqualTo(1); + } + + @Test + public void executeRemotely_responseWithoutResult_crashes() { + executionService.whenExecute(DUMMY_REQUEST).thenDone(ExecuteResponse.getDefaultInstance()); + + assertThrows( + IllegalStateException.class, + () -> executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP)); + + assertThat(executionService.getExecTimes()).isEqualTo(1); + } + + @Test + public void executeRemotely_retryWaitExecutionWhenUnauthenticated() + throws IOException, InterruptedException { + executionService.whenExecute(DUMMY_REQUEST).thenAck().finish(); + executionService.whenWaitExecution(DUMMY_REQUEST).thenAck().thenError(Code.UNAUTHENTICATED); + executionService.whenExecute(DUMMY_REQUEST).thenAck().thenDone(DUMMY_RESPONSE); + + ExecuteResponse response = + executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); + + assertThat(executionService.getExecTimes()).isEqualTo(2); + assertThat(executionService.getWaitTimes()).isEqualTo(1); + assertThat(response).isEqualTo(DUMMY_RESPONSE); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutorTestBase.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutorTestBase.java new file mode 100644 index 00000000000000..9094b5f5d45446 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcRemoteExecutorTestBase.java @@ -0,0 +1,330 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.remote; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import build.bazel.remote.execution.v2.ActionResult; +import build.bazel.remote.execution.v2.Digest; +import build.bazel.remote.execution.v2.ExecuteRequest; +import build.bazel.remote.execution.v2.ExecuteResponse; +import build.bazel.remote.execution.v2.ExecutionCapabilities; +import build.bazel.remote.execution.v2.OutputFile; +import build.bazel.remote.execution.v2.RequestMetadata; +import build.bazel.remote.execution.v2.ServerCapabilities; +import com.google.devtools.build.lib.remote.common.OperationObserver; +import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext; +import com.google.devtools.build.lib.remote.common.RemoteExecutionClient; +import com.google.devtools.build.lib.remote.grpc.ChannelConnectionFactory; +import com.google.devtools.build.lib.remote.options.RemoteOptions; +import com.google.devtools.build.lib.remote.util.TracingMetadataUtils; +import com.google.devtools.common.options.Options; +import com.google.longrunning.Operation; +import com.google.rpc.Code; +import io.grpc.ManagedChannel; +import io.grpc.Server; +import io.grpc.Status; +import io.grpc.inprocess.InProcessChannelBuilder; +import io.grpc.inprocess.InProcessServerBuilder; +import io.reactivex.rxjava3.core.Single; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** Base test class for {@link RemoteExecutionClient} gRPC implementations. */ +public abstract class GrpcRemoteExecutorTestBase { + + protected RemoteActionExecutionContext context; + protected FakeExecutionService executionService; + protected RemoteOptions remoteOptions; + private Server fakeServer; + protected RemoteExecutionClient executor; + + protected static final int MAX_RETRY_ATTEMPTS = 5; + + private static final OutputFile DUMMY_OUTPUT = + OutputFile.newBuilder() + .setPath("dummy.txt") + .setDigest( + Digest.newBuilder() + .setHash("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + .setSizeBytes(0) + .build()) + .build(); + + protected static final ExecuteRequest DUMMY_REQUEST = + ExecuteRequest.newBuilder() + .setInstanceName("dummy") + .setActionDigest( + Digest.newBuilder() + .setHash("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + .setSizeBytes(0) + .build()) + .build(); + + protected static final ExecuteResponse DUMMY_RESPONSE = + ExecuteResponse.newBuilder() + .setResult(ActionResult.newBuilder().addOutputFiles(DUMMY_OUTPUT).build()) + .build(); + + protected abstract RemoteExecutionClient createExecutionService( + ServerCapabilities caps, ReferenceCountedChannel channel) throws Exception; + + @Before + public void setUp() throws Exception { + context = RemoteActionExecutionContext.create(RequestMetadata.getDefaultInstance()); + + executionService = new FakeExecutionService(); + + String fakeServerName = "fake server for " + getClass(); + // Use a mutable service registry for later registering the service impl for each test case. + fakeServer = + InProcessServerBuilder.forName(fakeServerName) + .addService(executionService) + .directExecutor() + .build() + .start(); + + remoteOptions = Options.getDefaults(RemoteOptions.class); + remoteOptions.remoteMaxRetryAttempts = MAX_RETRY_ATTEMPTS; + + ReferenceCountedChannel channel = + new ReferenceCountedChannel( + new ChannelConnectionFactory() { + @Override + public Single create() { + ManagedChannel ch = + InProcessChannelBuilder.forName(fakeServerName) + .intercept(TracingMetadataUtils.newExecHeadersInterceptor(remoteOptions)) + .directExecutor() + .build(); + return Single.just(new ChannelConnection(ch)); + } + + @Override + public int maxConcurrency() { + return 100; + } + }); + + ServerCapabilities caps = + ServerCapabilities.newBuilder() + .setExecutionCapabilities( + ExecutionCapabilities.newBuilder().setExecEnabled(true).build()) + .build(); + + executor = createExecutionService(caps, channel); + } + + @After + public void tearDown() throws Exception { + fakeServer.shutdownNow(); + fakeServer.awaitTermination(); + + executor.close(); + } + + @Test + public void executeRemotely_smoke() throws Exception { + executionService.whenExecute(DUMMY_REQUEST).thenAck().thenAck().thenDone(DUMMY_RESPONSE); + + ExecuteResponse response = + executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); + + assertThat(response).isEqualTo(DUMMY_RESPONSE); + assertThat(executionService.getExecTimes()).isEqualTo(1); + } + + @Test + public void executeRemotely_errorInOperation_retryExecute() throws Exception { + executionService.whenExecute(DUMMY_REQUEST).thenError(new RuntimeException("Unavailable")); + executionService.whenExecute(DUMMY_REQUEST).thenError(Code.UNAVAILABLE); + executionService.whenExecute(DUMMY_REQUEST).thenAck().thenDone(DUMMY_RESPONSE); + + ExecuteResponse response = + executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); + + assertThat(executionService.getExecTimes()).isEqualTo(3); + assertThat(response).isEqualTo(DUMMY_RESPONSE); + } + + @Test + public void executeRemotely_errorInResponse_retryExecute() throws Exception { + executionService + .whenExecute(DUMMY_REQUEST) + .thenDone( + ExecuteResponse.newBuilder() + .setStatus(com.google.rpc.Status.newBuilder().setCode(Code.UNAVAILABLE_VALUE)) + .build()); + executionService.whenExecute(DUMMY_REQUEST).thenAck().thenDone(DUMMY_RESPONSE); + + ExecuteResponse response = + executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); + + assertThat(executionService.getExecTimes()).isEqualTo(2); + assertThat(response).isEqualTo(DUMMY_RESPONSE); + } + + @Test + public void executeRemotely_unretriableErrorInResponse_reportError() { + executionService + .whenExecute(DUMMY_REQUEST) + .thenDone( + ExecuteResponse.newBuilder() + .setStatus(com.google.rpc.Status.newBuilder().setCode(Code.INVALID_ARGUMENT_VALUE)) + .build()); + executionService.whenExecute(DUMMY_REQUEST).thenAck().thenDone(DUMMY_RESPONSE); + + IOException e = + assertThrows( + IOException.class, + () -> executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP)); + + assertThat(e).hasMessageThat().contains("INVALID_ARGUMENT"); + assertThat(executionService.getExecTimes()).isEqualTo(1); + } + + @Test + public void executeRemotely_retryExecuteAndFail() { + for (int i = 0; i <= MAX_RETRY_ATTEMPTS * 2; ++i) { + executionService.whenExecute(DUMMY_REQUEST).thenError(Code.UNAVAILABLE); + } + + IOException exception = + assertThrows( + IOException.class, + () -> executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP)); + + assertThat(executionService.getExecTimes()).isEqualTo(MAX_RETRY_ATTEMPTS + 1); + assertThat(exception).hasMessageThat().contains("UNAVAILABLE"); + } + + @Test + public void executeRemotely_executeAndWait() throws Exception { + executionService.whenExecute(DUMMY_REQUEST).thenAck().finish(); + executionService.whenWaitExecution(DUMMY_REQUEST).thenDone(DUMMY_RESPONSE); + + ExecuteResponse response = + executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); + + assertThat(executionService.getExecTimes()).isEqualTo(1); + assertThat(executionService.getWaitTimes()).isEqualTo(1); + assertThat(response).isEqualTo(DUMMY_RESPONSE); + } + + @Test + public void executeRemotely_executeAndRetryWait() throws Exception { + executionService.whenExecute(DUMMY_REQUEST).thenAck().finish(); + executionService.whenWaitExecution(DUMMY_REQUEST).thenDone(DUMMY_RESPONSE); + + ExecuteResponse response = + executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); + + assertThat(executionService.getExecTimes()).isEqualTo(1); + assertThat(executionService.getWaitTimes()).isEqualTo(1); + assertThat(response).isEqualTo(DUMMY_RESPONSE); + } + + @Test + public void executeRemotely_retryExecuteWhenUnauthenticated() + throws IOException, InterruptedException { + executionService.whenExecute(DUMMY_REQUEST).thenError(Code.UNAUTHENTICATED); + executionService.whenExecute(DUMMY_REQUEST).thenAck().thenDone(DUMMY_RESPONSE); + + ExecuteResponse response = + executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); + + assertThat(executionService.getExecTimes()).isEqualTo(2); + assertThat(response).isEqualTo(DUMMY_RESPONSE); + } + + @Test + public void executeRemotely_retryExecuteIfNotFound() throws IOException, InterruptedException { + executionService.whenExecute(DUMMY_REQUEST).thenAck().finish(); + executionService.whenWaitExecution(DUMMY_REQUEST).thenError(Code.NOT_FOUND); + executionService.whenExecute(DUMMY_REQUEST).thenAck().finish(); + executionService.whenWaitExecution(DUMMY_REQUEST).thenDone(DUMMY_RESPONSE); + + ExecuteResponse response = + executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); + + assertThat(executionService.getExecTimes()).isEqualTo(2); + assertThat(executionService.getWaitTimes()).isEqualTo(2); + assertThat(response).isEqualTo(DUMMY_RESPONSE); + } + + @Test + public void executeRemotely_retryExecuteOnFinish() throws IOException, InterruptedException { + executionService.whenExecute(DUMMY_REQUEST).thenAck().finish(); + executionService.whenWaitExecution(DUMMY_REQUEST).thenAck().finish(); + executionService.whenWaitExecution(DUMMY_REQUEST).thenAck().thenDone(DUMMY_RESPONSE); + + ExecuteResponse response = + executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); + + assertThat(executionService.getExecTimes()).isEqualTo(1); + assertThat(executionService.getWaitTimes()).isEqualTo(2); + assertThat(response).isEqualTo(DUMMY_RESPONSE); + } + + @Test + public void executeRemotely_notFoundLoop_reportError() { + for (int i = 0; i <= MAX_RETRY_ATTEMPTS; ++i) { + executionService.whenExecute(DUMMY_REQUEST).thenAck().finish(); + executionService.whenWaitExecution(DUMMY_REQUEST).thenAck().thenError(Code.NOT_FOUND); + } + + IOException e = + assertThrows( + IOException.class, + () -> executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP)); + + assertThat(e).hasCauseThat().isInstanceOf(ExecutionStatusException.class); + ExecutionStatusException executionStatusException = (ExecutionStatusException) e.getCause(); + assertThat(executionStatusException.getStatus().getCode()).isEqualTo(Status.Code.NOT_FOUND); + assertThat(executionService.getExecTimes()).isEqualTo(MAX_RETRY_ATTEMPTS + 1); + assertThat(executionService.getWaitTimes()).isEqualTo(MAX_RETRY_ATTEMPTS + 1); + } + + @Test + public void executeRemotely_notifyObserver() throws IOException, InterruptedException { + executionService.whenExecute(DUMMY_REQUEST).thenAck().thenDone(DUMMY_RESPONSE); + + List notified = new ArrayList<>(); + var unused = executor.executeRemotely(context, DUMMY_REQUEST, notified::add); + + assertThat(notified) + .containsExactly( + FakeExecutionService.ackOperation(DUMMY_REQUEST), + FakeExecutionService.doneOperation(DUMMY_REQUEST, DUMMY_RESPONSE)); + } + + @Test + public void executeRemotely_retryExecuteOnNoResultDoneOperation() + throws IOException, InterruptedException { + executionService.whenExecute(DUMMY_REQUEST).thenAck().thenError(Code.UNAVAILABLE); + executionService.whenExecute(DUMMY_REQUEST).thenAck().thenDone(DUMMY_RESPONSE); + + ExecuteResponse response = + executor.executeRemotely(context, DUMMY_REQUEST, OperationObserver.NO_OP); + + assertThat(executionService.getExecTimes()).isEqualTo(2); + assertThat(executionService.getWaitTimes()).isEqualTo(0); + assertThat(response).isEqualTo(DUMMY_RESPONSE); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/remote/UtilsTest.java b/src/test/java/com/google/devtools/build/lib/remote/UtilsTest.java index 6dcb2d53fdcb8b..c99d88bb2b9d3c 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/UtilsTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/UtilsTest.java @@ -36,14 +36,35 @@ public class UtilsTest { @Test - public void testGrpcAwareErrorMessages() { + public void testGrpcAwareErrorMessage() { IOException ioError = new IOException("io error"); IOException wrappedGrpcError = new IOException( "wrapped error", Status.ABORTED.withDescription("grpc error").asRuntimeException()); - assertThat(Utils.grpcAwareErrorMessage(ioError)).isEqualTo("io error"); - assertThat(Utils.grpcAwareErrorMessage(wrappedGrpcError)).isEqualTo("ABORTED: grpc error"); + assertThat(Utils.grpcAwareErrorMessage(ioError, /* verboseFailures= */ false)) + .isEqualTo("io error"); + assertThat(Utils.grpcAwareErrorMessage(wrappedGrpcError, /* verboseFailures= */ false)) + .isEqualTo("ABORTED: grpc error"); + } + + @Test + public void testGrpcAwareErrorMessage_verboseFailures() { + IOException ioError = new IOException("io error"); + IOException wrappedGrpcError = + new IOException( + "wrapped error", Status.ABORTED.withDescription("grpc error").asRuntimeException()); + + assertThat(Utils.grpcAwareErrorMessage(ioError, /* verboseFailures= */ true)) + .startsWith( + "io error\n" + + "java.io.IOException: io error\n" + + "\tat com.google.devtools.build.lib.remote.UtilsTest.testGrpcAwareErrorMessage_verboseFailures"); + assertThat(Utils.grpcAwareErrorMessage(wrappedGrpcError, /* verboseFailures= */ true)) + .startsWith( + "ABORTED: grpc error\n" + + "java.io.IOException: wrapped error\n" + + "\tat com.google.devtools.build.lib.remote.UtilsTest.testGrpcAwareErrorMessage_verboseFailures"); } @Test From 150eaaf23b3fd69beeca91f5077091778ac531a9 Mon Sep 17 00:00:00 2001 From: Brentley Jones Date: Tue, 25 Jul 2023 08:00:42 -0500 Subject: [PATCH 003/125] Move BazelFileSystemModule into bazel package (#19043) Currently, it is included in runtime library with glob which is shared between bazel and blaze. Moving it to bazel package to prevent from packaging it when building blaze. An upcoming change (in https://github.com/bazelbuild/bazel/pull/18784) will package BLAKE3 to this module and we don't want it in blaze. PiperOrigin-RevId: 547516042 Change-Id: I673fa3d6824d56c85e85b02d13b4eb56571edda9 (cherry picked from commit 6d6bbdcd7ac7e8130741417d891e641495b046b2) Co-authored-by: Googler --- .../com/google/devtools/build/lib/bazel/BUILD | 20 +++++++++++++++++++ .../devtools/build/lib/bazel/Bazel.java | 2 +- .../BazelFileSystemModule.java | 4 +++- 3 files changed, 24 insertions(+), 2 deletions(-) rename src/main/java/com/google/devtools/build/lib/{runtime => bazel}/BazelFileSystemModule.java (95%) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/BUILD index bd7402b7f9d77a..a131ab63a6d3c4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/BUILD @@ -131,10 +131,30 @@ java_library( ], ) +java_library( + name = "bazel_filesystem_module", + srcs = ["BazelFileSystemModule.java"], + deps = [ + "//src/main/java/com/google/devtools/build/lib:runtime", + "//src/main/java/com/google/devtools/build/lib/jni", + "//src/main/java/com/google/devtools/build/lib/unix", + "//src/main/java/com/google/devtools/build/lib/util:abrupt_exit_exception", + "//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code", + "//src/main/java/com/google/devtools/build/lib/util:os", + "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", + "//src/main/java/com/google/devtools/build/lib/windows", + "//src/main/java/com/google/devtools/common/options", + "//src/main/protobuf:failure_details_java_proto", + "//third_party:guava", + ], +) + java_library( name = "main", srcs = ["Bazel.java"], deps = [ + ":bazel_filesystem_module", ":builtin_command_module", ":modules", ":repository_module", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java index 7fb055d87e2c30..2590d91b2681f2 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/Bazel.java @@ -49,7 +49,7 @@ public final class Bazel { com.google.devtools.build.lib.runtime.MemoryPressureModule.class, com.google.devtools.build.lib.platform.SleepPreventionModule.class, com.google.devtools.build.lib.platform.SystemSuspensionModule.class, - com.google.devtools.build.lib.runtime.BazelFileSystemModule.class, + BazelFileSystemModule.class, com.google.devtools.build.lib.runtime.mobileinstall.MobileInstallModule.class, com.google.devtools.build.lib.bazel.BazelWorkspaceStatusModule.class, com.google.devtools.build.lib.bazel.BazelDiffAwarenessModule.class, diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BazelFileSystemModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelFileSystemModule.java similarity index 95% rename from src/main/java/com/google/devtools/build/lib/runtime/BazelFileSystemModule.java rename to src/main/java/com/google/devtools/build/lib/bazel/BazelFileSystemModule.java index 58907a7c0ec662..1027f3a6de3e4c 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/BazelFileSystemModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelFileSystemModule.java @@ -11,12 +11,14 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -package com.google.devtools.build.lib.runtime; +package com.google.devtools.build.lib.bazel; import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.base.Strings; import com.google.devtools.build.lib.jni.JniLoader; +import com.google.devtools.build.lib.runtime.BlazeModule; +import com.google.devtools.build.lib.runtime.BlazeServerStartupOptions; import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; import com.google.devtools.build.lib.server.FailureDetails.Filesystem; import com.google.devtools.build.lib.server.FailureDetails.Filesystem.Code; From 2b76e16dd63a04e2815225f6fc4880791e52b87f Mon Sep 17 00:00:00 2001 From: Brentley Jones Date: Tue, 25 Jul 2023 08:39:49 -0500 Subject: [PATCH 004/125] Fix a bug where frozen targets list was mutated while expanding env attribute (#19053) for cc_test. Add a test covering the behavior. PiperOrigin-RevId: 504294442 Change-Id: If9f96b631ca958e746613563c56103044c973277 (cherry picked from commit a63d8ebb84caba9d1741a9d9fec725a7db52e25e) Co-authored-by: Googler --- .../builtins_bzl/common/cc/cc_helper.bzl | 5 +++-- .../google/devtools/build/lib/rules/cpp/BUILD | 1 + .../build/lib/rules/cpp/CcCommonTest.java | 16 ++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl index 1c40b6f70285dc..91e629b0d8baaa 100644 --- a/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/cc_helper.bzl @@ -713,9 +713,10 @@ def _expand_nested_variable(ctx, additional_vars, exp, execpath = True, targets if not execpath: if exp.startswith("location"): exp = exp.replace("location", "rootpath", 1) + data_targets = [] if ctx.attr.data != None: - targets.extend(ctx.attr.data) - return ctx.expand_location("$({})".format(exp), targets = targets) + data_targets = ctx.attr.data + return ctx.expand_location("$({})".format(exp), targets = targets + data_targets) # Recursively expand nested make variables, but since there is no recursion # in Starlark we will do it via for loop. diff --git a/src/test/java/com/google/devtools/build/lib/rules/cpp/BUILD b/src/test/java/com/google/devtools/build/lib/rules/cpp/BUILD index c89878351c2793..3eff8b98a47a71 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/cpp/BUILD +++ b/src/test/java/com/google/devtools/build/lib/rules/cpp/BUILD @@ -146,6 +146,7 @@ java_test( "//src/main/java/com/google/devtools/build/lib/actions:artifacts", "//src/main/java/com/google/devtools/build/lib/analysis:analysis_cluster", "//src/main/java/com/google/devtools/build/lib/analysis:configured_target", + "//src/main/java/com/google/devtools/build/lib/analysis:run_environment_info", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/collect/nestedset", "//src/main/java/com/google/devtools/build/lib/rules/cpp", diff --git a/src/test/java/com/google/devtools/build/lib/rules/cpp/CcCommonTest.java b/src/test/java/com/google/devtools/build/lib/rules/cpp/CcCommonTest.java index 5ff2ea45df325b..3ae7838b5cf15d 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/cpp/CcCommonTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/cpp/CcCommonTest.java @@ -27,6 +27,7 @@ import com.google.devtools.build.lib.analysis.AnalysisUtils; import com.google.devtools.build.lib.analysis.ConfiguredTarget; import com.google.devtools.build.lib.analysis.OutputGroupInfo; +import com.google.devtools.build.lib.analysis.RunEnvironmentInfo; import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget; import com.google.devtools.build.lib.analysis.util.AnalysisMock; import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; @@ -805,6 +806,21 @@ public void testExpandedLinkopts() throws Exception { .getPathString())); } + @Test + public void testExpandedEnv() throws Exception { + scratch.file( + "a/BUILD", + "genrule(name = 'linker', cmd='generate', outs=['a.lds'])", + "cc_test(", + " name='bin_test',", + " srcs=['b.cc'],", + " env={'SOME_KEY': '-Wl,@$(location a.lds)'},", + " deps=['a.lds'])"); + ConfiguredTarget starlarkTarget = getConfiguredTarget("//a:bin_test"); + RunEnvironmentInfo provider = starlarkTarget.get(RunEnvironmentInfo.PROVIDER); + assertThat(provider.getEnvironment()).containsEntry("SOME_KEY", "-Wl,@a/a.lds"); + } + @Test public void testProvidesLinkerScriptToLinkAction() throws Exception { scratch.file( From b73b0753ee3aa7a6ccbfacbaa10e468e2abd6029 Mon Sep 17 00:00:00 2001 From: keertk Date: Tue, 25 Jul 2023 12:44:25 -0700 Subject: [PATCH 005/125] Mark isolated extension usages as experimental (#19065) The `isolate` parameter of `use_extension` and the `is_isolated` field on `module_ctx` are only available with the `--experimental_isolated_extension_usages` flag. Closes #19021. PiperOrigin-RevId: 550582124 Change-Id: I7486a3b344ca9d29614c26044f1943bfedfcc654 Co-authored-by: Fabian Meumertzheim --- .../bazel/bzlmod/ModuleExtensionContext.java | 7 +++-- .../lib/bazel/bzlmod/ModuleFileGlobals.java | 8 +++-- .../semantics/BuildLanguageOptions.java | 14 +++++++++ .../bzlmod/ModuleExtensionResolutionTest.java | 5 +++- .../bazel/bzlmod/ModuleFileFunctionTest.java | 30 +++++++++++++++++++ .../packages/semantics/ConsistencyTest.java | 2 ++ .../py/bazel/bzlmod/bazel_lockfile_test.py | 1 + 7 files changed, 62 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionContext.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionContext.java index b7a392673c2db9..456148a1f4db63 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionContext.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionContext.java @@ -128,8 +128,11 @@ public boolean isDevDependency(TypeCheckedTag tag) { name = "is_isolated", doc = "Whether this particular usage of the extension had isolate = True " - + "specified and is thus isolated from all other usages.", - structField = true) + + "specified and is thus isolated from all other usages." + + "

This field is currently experimental and only available with the flag " + + "--experimental_isolated_extension_usages.", + structField = true, + enableOnlyWithFlag = "-experimental_isolated_extension_usages") public boolean isIsolated() { return extensionId.getIsolationKey().isPresent(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java index 3c45f89a4f17db..e44a3b6e70d450 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java @@ -435,10 +435,14 @@ public void registerToolchains(boolean devDependency, Sequence toolchainLabel + "usages, both in this and other modules. Tags created for this usage do not " + "affect other usages and the repositories generated by the extension for " + "this usage will be distinct from all other repositories generated by the " - + "extension.", + + "extension." + + "

This parameter is currently experimental and only available with the " + + "flag --experimental_isolated_extension_usages.", named = true, positional = false, - defaultValue = "False"), + defaultValue = "False", + enableOnlyWithFlag = "-experimental_isolated_extension_usages", + valueWhenDisabled = "False"), }, useStarlarkThread = true) public ModuleExtensionProxy useExtension( diff --git a/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java b/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java index 054166f0687d15..4e3119916db530 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java +++ b/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java @@ -199,6 +199,17 @@ public final class BuildLanguageOptions extends OptionsBase { + " WORKSPACE. See https://bazel.build/docs/bzlmod for more information.") public boolean enableBzlmod; + @Option( + name = "experimental_isolated_extension_usages", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.STARLARK_SEMANTICS, + effectTags = OptionEffectTag.LOADING_AND_ANALYSIS, + help = + "If true, enables the isolate parameter in the use_extension" + + " function.") + public boolean experimentalIsolatedExtensionUsages; + @Option( name = "experimental_java_proto_library_default_has_services", defaultValue = "true", @@ -669,6 +680,7 @@ public StarlarkSemantics toStarlarkSemantics() { .setBool( EXPERIMENTAL_ENABLE_ANDROID_MIGRATION_APIS, experimentalEnableAndroidMigrationApis) .setBool(ENABLE_BZLMOD, enableBzlmod) + .setBool(EXPERIMENTAL_ISOLATED_EXTENSION_USAGES, experimentalIsolatedExtensionUsages) .setBool( EXPERIMENTAL_JAVA_PROTO_LIBRARY_DEFAULT_HAS_SERVICES, experimentalJavaProtoLibraryDefaultHasServices) @@ -761,6 +773,8 @@ public StarlarkSemantics toStarlarkSemantics() { public static final String EXPERIMENTAL_ENABLE_ANDROID_MIGRATION_APIS = "-experimental_enable_android_migration_apis"; public static final String ENABLE_BZLMOD = "-enable_bzlmod"; + public static final String EXPERIMENTAL_ISOLATED_EXTENSION_USAGES = + "-experimental_isolated_extension_usages"; public static final String EXPERIMENTAL_JAVA_PROTO_LIBRARY_DEFAULT_HAS_SERVICES = "+experimental_java_proto_library_default_has_services"; public static final String INCOMPATIBLE_EXISTING_RULES_IMMUTABLE_VIEW = diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java index 3e8bb0920f5b2a..675a9ce4f7816c 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java @@ -263,7 +263,10 @@ public void setup() throws Exception { PrecomputedValue.STARLARK_SEMANTICS.set( differencer, - StarlarkSemantics.builder().setBool(BuildLanguageOptions.ENABLE_BZLMOD, true).build()); + StarlarkSemantics.builder() + .setBool(BuildLanguageOptions.ENABLE_BZLMOD, true) + .setBool(BuildLanguageOptions.EXPERIMENTAL_ISOLATED_EXTENSION_USAGES, true) + .build()); RepositoryDelegatorFunction.REPOSITORY_OVERRIDES.set(differencer, ImmutableMap.of()); RepositoryDelegatorFunction.DEPENDENCY_FOR_UNCONDITIONAL_FETCHING.set( differencer, RepositoryDelegatorFunction.DONT_FETCH_UNCONDITIONALLY); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java index 3ee5f0d59ff4be..35b09dcdf214fd 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java @@ -1049,6 +1049,13 @@ public void module_calledLate() throws Exception { @Test public void isolatedUsageWithoutImports() throws Exception { + PrecomputedValue.STARLARK_SEMANTICS.set( + differencer, + StarlarkSemantics.builder() + .setBool(BuildLanguageOptions.ENABLE_BZLMOD, true) + .setBool(BuildLanguageOptions.EXPERIMENTAL_ISOLATED_EXTENSION_USAGES, true) + .build()); + scratch.file( rootDirectory.getRelative("MODULE.bazel").getPathString(), "isolated_ext = use_extension('//:extensions.bzl', 'my_ext', isolate = True)"); @@ -1069,6 +1076,13 @@ public void isolatedUsageWithoutImports() throws Exception { @Test public void isolatedUsageNotExported() throws Exception { + PrecomputedValue.STARLARK_SEMANTICS.set( + differencer, + StarlarkSemantics.builder() + .setBool(BuildLanguageOptions.ENABLE_BZLMOD, true) + .setBool(BuildLanguageOptions.EXPERIMENTAL_ISOLATED_EXTENSION_USAGES, true) + .build()); + scratch.file( rootDirectory.getRelative("MODULE.bazel").getPathString(), "use_extension('//:extensions.bzl', 'my_ext', isolate = True)"); @@ -1081,4 +1095,20 @@ public void isolatedUsageNotExported() throws Exception { "Isolated extension usage at /workspace/MODULE.bazel:1:14 must be assigned to a " + "top-level variable"); } + + @Test + public void isolatedUsage_notEnabled() throws Exception { + scratch.file( + rootDirectory.getRelative("MODULE.bazel").getPathString(), + "use_extension('//:extensions.bzl', 'my_ext', isolate = True)"); + FakeRegistry registry = registryFactory.newFakeRegistry("/foo"); + ModuleFileFunction.REGISTRIES.set(differencer, ImmutableList.of(registry.getUrl())); + + reporter.removeHandler(failFastHandler); // expect failures + evaluator.evaluate(ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); + assertContainsEvent( + "Error in use_extension: in call to use_extension(), parameter 'isolate' is experimental " + + "and thus unavailable with the current flags. It may be enabled by setting " + + "--experimental_isolated_extension_usages"); + } } diff --git a/src/test/java/com/google/devtools/build/lib/packages/semantics/ConsistencyTest.java b/src/test/java/com/google/devtools/build/lib/packages/semantics/ConsistencyTest.java index 4cf733d5057d12..246bbe7e4b67ee 100644 --- a/src/test/java/com/google/devtools/build/lib/packages/semantics/ConsistencyTest.java +++ b/src/test/java/com/google/devtools/build/lib/packages/semantics/ConsistencyTest.java @@ -128,6 +128,7 @@ private static BuildLanguageOptions buildRandomOptions(Random rand) throws Excep "--experimental_bzl_visibility=" + rand.nextBoolean(), "--experimental_enable_android_migration_apis=" + rand.nextBoolean(), "--enable_bzlmod=" + rand.nextBoolean(), + "--experimental_isolated_extension_usages=" + rand.nextBoolean(), "--experimental_google_legacy_api=" + rand.nextBoolean(), "--experimental_platforms_api=" + rand.nextBoolean(), "--incompatible_allow_tags_propagation=" + rand.nextBoolean(), // flag, Java names differ @@ -170,6 +171,7 @@ private static StarlarkSemantics buildRandomSemantics(Random rand) { .setBool( BuildLanguageOptions.EXPERIMENTAL_ENABLE_ANDROID_MIGRATION_APIS, rand.nextBoolean()) .setBool(BuildLanguageOptions.ENABLE_BZLMOD, rand.nextBoolean()) + .setBool(BuildLanguageOptions.EXPERIMENTAL_ISOLATED_EXTENSION_USAGES, rand.nextBoolean()) .setBool(BuildLanguageOptions.EXPERIMENTAL_GOOGLE_LEGACY_API, rand.nextBoolean()) .setBool(BuildLanguageOptions.EXPERIMENTAL_PLATFORMS_API, rand.nextBoolean()) .setBool(BuildLanguageOptions.EXPERIMENTAL_ALLOW_TAGS_PROPAGATION, rand.nextBoolean()) diff --git a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py index 76f603b48e9c4c..f145c633d301b5 100644 --- a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py +++ b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py @@ -45,6 +45,7 @@ def setUp(self): # In ipv6 only network, this has to be enabled. # 'startup --host_jvm_args=-Djava.net.preferIPv6Addresses=true', 'common --enable_bzlmod', + 'common --experimental_isolated_extension_usages', 'common --registry=' + self.main_registry.getURL(), # We need to have BCR here to make sure built-in modules like # bazel_tools can work. From 9a4607617437a7b526f1c88f0e2fc2fe198f52b9 Mon Sep 17 00:00:00 2001 From: "Ian (Hee) Cha" Date: Wed, 26 Jul 2023 04:57:55 -0700 Subject: [PATCH 006/125] Add the remote_require_cached flag (#19075) When set to true, this flag causes Bazel to abort the build whenever it encounters an action that is not cached. This is very useful when trying to troubleshoot action caching issues across machines because it allows running a build on one and having it fail on another as soon as there is a problem without tainting what already exists in the cache. My workflow is to essentially do: 1. Machine 1: bazel clean 2. Machine 1: bazel build ... 3. Machine 2: bazel clean 4. Machine 2: bazel build --remote_require_cached ... which makes step 4 fail on the first action that wasn't cached as expected. Then I can address that problem and re-run step 4 to encounter the next issue. Closes #18942. PiperOrigin-RevId: 549242966 Change-Id: Ib46a2eb8cce6f4444968882e99c21284fc6bc4f8 Co-authored-by: Julio Merino --- .../build/lib/remote/RemoteSpawnRunner.java | 16 ++++++++++ .../lib/remote/options/RemoteOptions.java | 12 ++++++++ .../bazel/remote/remote_execution_test.sh | 30 +++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java index 4be0774599b770..b23b585ab24e51 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java @@ -237,6 +237,22 @@ public SpawnResult exec(Spawn spawn, SpawnExecutionContext context) return execLocallyAndUploadOrFail(action, spawn, context, uploadLocalResults, e); } + if (remoteOptions.remoteRequireCached) { + return new SpawnResult.Builder() + .setStatus(SpawnResult.Status.EXECUTION_DENIED) + .setExitCode(1) + .setFailureMessage( + "Action must be cached due to --experimental_remote_require_cached but it is not") + .setFailureDetail( + FailureDetail.newBuilder() + .setSpawn( + FailureDetails.Spawn.newBuilder() + .setCode(FailureDetails.Spawn.Code.EXECUTION_DENIED)) + .build()) + .setRunnerName("remote") + .build(); + } + AtomicBoolean useCachedResult = new AtomicBoolean(acceptCachedResult); AtomicBoolean forceUploadInput = new AtomicBoolean(false); try { diff --git a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java index 793eccdfa51756..092d5cafdd31ff 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java +++ b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java @@ -250,6 +250,18 @@ public String getTypeDescription() { help = "Whether to accept remotely cached action results.") public boolean remoteAcceptCached; + @Option( + name = "experimental_remote_require_cached", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.REMOTE, + effectTags = {OptionEffectTag.UNKNOWN}, + help = + "If set to true, enforce that all actions that can run remotely are cached, or else " + + "fail the build. This is useful to troubleshoot non-determinism issues as it " + + "allows checking whether actions that should be cached are actually cached " + + "without spuriously injecting new results into the cache.") + public boolean remoteRequireCached; + @Option( name = "remote_local_fallback", defaultValue = "false", diff --git a/src/test/shell/bazel/remote/remote_execution_test.sh b/src/test/shell/bazel/remote/remote_execution_test.sh index ff06d5d7c5cfb2..ffe8f428457a63 100755 --- a/src/test/shell/bazel/remote/remote_execution_test.sh +++ b/src/test/shell/bazel/remote/remote_execution_test.sh @@ -1253,6 +1253,36 @@ EOF expect_not_log "1 local" } +function test_require_cached() { + mkdir -p a + cat > a/BUILD <<'EOF' +genrule( + name = "foo", + srcs = ["foo.in"], + outs = ["foo.out"], + cmd = "cp \"$<\" \"$@\"", +) +EOF + + echo "input 1" >a/foo.in + bazel build \ + --remote_executor=grpc://localhost:${worker_port} \ + //a:foo >& $TEST_log || fail "Failed to build //a:foo" + + expect_log "1 remote" + + echo "input 2" >a/foo.in + if bazel build \ + --remote_executor=grpc://localhost:${worker_port} \ + --experimental_remote_require_cached \ + //a:foo >& $TEST_log; then + fail "Build of //a:foo succeeded but it should have failed" + fi + + expect_log "Action must be cached due to --experimental_remote_require_cached but it is not" + expect_not_log "remote cache hit" +} + function test_nobuild_runfile_links() { mkdir data && echo "hello" > data/hello && echo "world" > data/world cat > WORKSPACE < Date: Wed, 26 Jul 2023 10:34:12 -0700 Subject: [PATCH 007/125] [6.3.1] Advertise CcInfo from cc_import (#19086) (#19088) * Advertise CcInfo from cc_import Fixes #19056 * Update BUILD.builtin_test Co-authored-by: oquenchil <23365806+oquenchil@users.noreply.github.com> --- src/main/starlark/builtins_bzl/common/cc/cc_import.bzl | 1 + .../test_cc_shared_library/BUILD.builtin_test | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_import.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_import.bzl index babfb2b40c989e..b7c7ff8df636c7 100644 --- a/src/main/starlark/builtins_bzl/common/cc/cc_import.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/cc_import.bzl @@ -220,6 +220,7 @@ cc_import = rule( ), "_cc_toolchain": attr.label(default = "@" + semantics.get_repo() + "//tools/cpp:current_cc_toolchain"), }, + provides = [CcInfo], toolchains = cc_helper.use_cpp_toolchain(), fragments = ["cpp"], incompatible_use_toolchain_transition = True, diff --git a/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/BUILD.builtin_test b/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/BUILD.builtin_test index 952dea0c426454..196231642e74e2 100644 --- a/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/BUILD.builtin_test +++ b/src/main/starlark/tests/builtins_bzl/cc/cc_shared_library/test_cc_shared_library/BUILD.builtin_test @@ -405,12 +405,10 @@ paths_test( name = "path_matching_test", ) -cc_library( +cc_import( name = "prebuilt", hdrs = ["direct_so_file_cc_lib.h"], - srcs = [ - ":just_main_output", - ], + shared_library = ":just_main_output", ) filegroup( From c24329a1e5aaf6cbefb47cbc8b2d3307f4f36ead Mon Sep 17 00:00:00 2001 From: keertk Date: Wed, 26 Jul 2023 13:30:48 -0700 Subject: [PATCH 008/125] Update java_tools version to 12.6 (#19092) --- distdir_deps.bzl | 50 ++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/distdir_deps.bzl b/distdir_deps.bzl index ac773f6478d239..b896f1b24eaf14 100644 --- a/distdir_deps.bzl +++ b/distdir_deps.bzl @@ -382,80 +382,80 @@ DIST_DEPS = { "remote_java_tools_test", "remote_java_tools_for_testing", ], - "archive": "java_tools-v12.5.zip", - "sha256": "942b3d88ebd785a5face38049077a1f8dab5a3500f5ebd0c0df090244acc4e16", + "archive": "java_tools-v12.6.zip", + "sha256": "f58a358ca694a41416a9b6a92b852935ad301d8882e5d22f4f11134f035317d5", "urls": [ - "https://mirror.bazel.build/bazel_java_tools/releases/java/v12.5/java_tools-v12.5.zip", - "https://github.com/bazelbuild/java_tools/releases/download/java_v12.5/java_tools-v12.5.zip", + "https://mirror.bazel.build/bazel_java_tools/releases/java/v12.6/java_tools-v12.6.zip", + "https://github.com/bazelbuild/java_tools/releases/download/java_v12.6/java_tools-v12.6.zip", ], "used_in": [ "test_WORKSPACE_files", ], - "package_version": "12.5", + "package_version": "12.6", }, "remote_java_tools_linux": { "aliases": [ "remote_java_tools_test_linux", "remote_java_tools_linux_for_testing", ], - "archive": "java_tools_linux-v12.5.zip", - "sha256": "45dda5441b46385e8ec95d3bc4a04b9337a6ff837bb41bdaa6247d8d36edceae", + "archive": "java_tools_linux-v12.6.zip", + "sha256": "64294e91fe940c77e6d35818b4c3a1f07d78e33add01e330188d907032687066", "urls": [ - "https://mirror.bazel.build/bazel_java_tools/releases/java/v12.5/java_tools_linux-v12.5.zip", - "https://github.com/bazelbuild/java_tools/releases/download/java_v12.5/java_tools_linux-v12.5.zip", + "https://mirror.bazel.build/bazel_java_tools/releases/java/v12.6/java_tools_linux-v12.6.zip", + "https://github.com/bazelbuild/java_tools/releases/download/java_v12.6/java_tools_linux-v12.6.zip", ], "used_in": [ "test_WORKSPACE_files", ], - "package_version": "12.5", + "package_version": "12.6", }, "remote_java_tools_windows": { "aliases": [ "remote_java_tools_test_windows", "remote_java_tools_windows_for_testing", ], - "archive": "java_tools_windows-v12.5.zip", - "sha256": "0b319cf762e256133f8d0f5f99fd7d35ca4cf00f35e7c0e8aea1b9fcd9cf4fb0", + "archive": "java_tools_windows-v12.6.zip", + "sha256": "63f727d44011b8c504bb4e6d89c2cd982278efb34dae8629687e9483d8f7d62d", "urls": [ - "https://mirror.bazel.build/bazel_java_tools/releases/java/v12.5/java_tools_windows-v12.5.zip", - "https://github.com/bazelbuild/java_tools/releases/download/java_v12.5/java_tools_windows-v12.5.zip", + "https://mirror.bazel.build/bazel_java_tools/releases/java/v12.6/java_tools_windows-v12.6.zip", + "https://github.com/bazelbuild/java_tools/releases/download/java_v12.6/java_tools_windows-v12.6.zip", ], "used_in": [ "test_WORKSPACE_files", ], - "package_version": "12.5", + "package_version": "12.6", }, "remote_java_tools_darwin_x86_64": { "aliases": [ "remote_java_tools_test_darwin_x86_64", "remote_java_tools_darwin_x86_64_for_testing", ], - "archive": "java_tools_darwin_x86_64-v12.5.zip", - "sha256": "7e96d0310222e9c27e4c87987978c0c59a0acb97cebdbd838ff652a13abbed77", + "archive": "java_tools_darwin_x86_64-v12.6.zip", + "sha256": "c6545e82e543cb5775d3b8909d6270b5f481864b5ff083d20bfa5dcf77ac3ef7", "urls": [ - "https://mirror.bazel.build/bazel_java_tools/releases/java/v12.5/java_tools_darwin_x86_64-v12.5.zip", - "https://github.com/bazelbuild/java_tools/releases/download/java_v12.5/java_tools_darwin_x86_64-v12.5.zip", + "https://mirror.bazel.build/bazel_java_tools/releases/java/v12.6/java_tools_darwin_x86_64-v12.6.zip", + "https://github.com/bazelbuild/java_tools/releases/download/java_v12.6/java_tools_darwin_x86_64-v12.6.zip", ], "used_in": [ "test_WORKSPACE_files", ], - "package_version": "12.5", + "package_version": "12.6", }, "remote_java_tools_darwin_arm64": { "aliases": [ "remote_java_tools_test_darwin_arm64", "remote_java_tools_darwin_arm64_for_testing", ], - "archive": "java_tools_darwin_arm64-v12.5.zip", - "sha256": "5fb927b24043dd79010b54be31ec5f18b38ee9dbd9fd03d8353232431a7e2392", + "archive": "java_tools_darwin_arm64-v12.6.zip", + "sha256": "c6ffcaf87965c436cc86fc0e9673dafc97c0761efae8225ad2691cf6cfe3d87a", "urls": [ - "https://mirror.bazel.build/bazel_java_tools/releases/java/v12.5/java_tools_darwin_arm64-v12.5.zip", - "https://github.com/bazelbuild/java_tools/releases/download/java_v12.5/java_tools_darwin_arm64-v12.5.zip", + "https://mirror.bazel.build/bazel_java_tools/releases/java/v12.6/java_tools_darwin_arm64-v12.6.zip", + "https://github.com/bazelbuild/java_tools/releases/download/java_v12.6/java_tools_darwin_arm64-v12.6.zip", ], "used_in": [ "test_WORKSPACE_files", ], - "package_version": "12.5", + "package_version": "12.6", }, "remotejdk11_linux": { "aliases": [ From b7d63136889a65cbf013648d88573210e6fa0357 Mon Sep 17 00:00:00 2001 From: Brentley Jones Date: Thu, 27 Jul 2023 07:55:05 -0500 Subject: [PATCH 009/125] Set the digest_function field as part of all relevant gRPC requests (#19049) In the following PR we extended most of the REv2 *Request messages to take an explicit digest function: https://github.com/bazelbuild/remote-apis/pull/235 This eliminates the ambiguity between digest functions that have the same hash length (e.g., MD5 and MURMUR3, SHA256 and SHA256TREE). This change extends the REv2 client in Bazel to set the digest_function field explicitly, so that the server does not need to use any heuristics to pick a digest function when processing requests. As we are now going to call DigestUtil.getDigestFunction() far more frequently, alter DigestUtil to precompute the value upon construction. This change was tested by letting Bazel forward all traffic through an instance of bb_clientd that was patched up to require the use of `digest_function`. PiperOrigin-RevId: 520074622 Change-Id: Ie92dc368c502b9bc3fddef56a5eb0a238760f673 (cherry picked from commit 0a8380bec9dceae1bffabddcccd459e82d8674b1) Signed-off-by: Brentley Jones Co-authored-by: Ed Schouten --- .../devtools/build/lib/remote/GrpcCacheClient.java | 7 ++++++- .../build/lib/remote/RemoteExecutionService.java | 1 + .../lib/remote/RemoteRepositoryRemoteExecutor.java | 1 + .../devtools/build/lib/remote/util/DigestUtil.java | 10 ++++++++-- .../devtools/build/lib/remote/GrpcCacheClientTest.java | 2 ++ 5 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/GrpcCacheClient.java b/src/main/java/com/google/devtools/build/lib/remote/GrpcCacheClient.java index ce0e917837f0c5..470d3845dc3273 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/GrpcCacheClient.java +++ b/src/main/java/com/google/devtools/build/lib/remote/GrpcCacheClient.java @@ -117,6 +117,7 @@ private int computeMaxMissingBlobsDigestsPerMessage() { final int overhead = FindMissingBlobsRequest.newBuilder() .setInstanceName(options.remoteInstanceName) + .setDigestFunction(digestUtil.getDigestFunction()) .build() .getSerializedSize(); final int tagSize = @@ -185,7 +186,9 @@ public ListenableFuture> findMissingDigests( } // Need to potentially split the digests into multiple requests. FindMissingBlobsRequest.Builder requestBuilder = - FindMissingBlobsRequest.newBuilder().setInstanceName(options.remoteInstanceName); + FindMissingBlobsRequest.newBuilder() + .setInstanceName(options.remoteInstanceName) + .setDigestFunction(digestUtil.getDigestFunction()); List> getMissingDigestCalls = new ArrayList<>(); for (Digest digest : digests) { requestBuilder.addBlobDigests(digest); @@ -260,6 +263,7 @@ public ListenableFuture downloadActionResult( GetActionResultRequest request = GetActionResultRequest.newBuilder() .setInstanceName(options.remoteInstanceName) + .setDigestFunction(digestUtil.getDigestFunction()) .setActionDigest(actionKey.getDigest()) .setInlineStderr(inlineOutErr) .setInlineStdout(inlineOutErr) @@ -289,6 +293,7 @@ public ListenableFuture uploadActionResult( .updateActionResult( UpdateActionResultRequest.newBuilder() .setInstanceName(options.remoteInstanceName) + .setDigestFunction(digestUtil.getDigestFunction()) .setActionDigest(actionKey.getDigest()) .setActionResult(actionResult) .build())), diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java index a2cf4ae1597505..c3545b669c338a 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java @@ -1482,6 +1482,7 @@ public RemoteActionResult executeRemotely( ExecuteRequest.Builder requestBuilder = ExecuteRequest.newBuilder() .setInstanceName(remoteOptions.remoteInstanceName) + .setDigestFunction(digestUtil.getDigestFunction()) .setActionDigest(action.getActionKey().getDigest()) .setSkipCacheLookup(!acceptCachedResult); if (remoteOptions.remoteResultCachePriority != 0) { diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteRepositoryRemoteExecutor.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteRepositoryRemoteExecutor.java index 69f521e84da95f..3a731f72709be2 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteRepositoryRemoteExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteRepositoryRemoteExecutor.java @@ -167,6 +167,7 @@ public ExecutionResult execute( ExecuteRequest.newBuilder() .setActionDigest(actionDigest) .setInstanceName(remoteInstanceName) + .setDigestFunction(digestUtil.getDigestFunction()) .setSkipCacheLookup(!acceptCached) .build(); diff --git a/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java b/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java index 16055f13a7ee3d..76b258443890ca 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java +++ b/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java @@ -35,14 +35,15 @@ public class DigestUtil { private final XattrProvider xattrProvider; private final DigestHashFunction hashFn; + private final DigestFunction.Value digestFunction; public DigestUtil(XattrProvider xattrProvider, DigestHashFunction hashFn) { this.xattrProvider = xattrProvider; this.hashFn = hashFn; + this.digestFunction = getDigestFunctionFromHashFunction(hashFn); } - /** Returns the currently used digest function. */ - public DigestFunction.Value getDigestFunction() { + private static DigestFunction.Value getDigestFunctionFromHashFunction(DigestHashFunction hashFn) { for (String name : hashFn.getNames()) { try { return DigestFunction.Value.valueOf(name); @@ -53,6 +54,11 @@ public DigestFunction.Value getDigestFunction() { return DigestFunction.Value.UNKNOWN; } + /** Returns the currently used digest function. */ + public DigestFunction.Value getDigestFunction() { + return digestFunction; + } + public Digest compute(byte[] blob) { return buildDigest(hashFn.getHashFunction().hashBytes(blob).toString(), blob.length); } diff --git a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java index 24212e84978627..41fcca47ada4ce 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/GrpcCacheClientTest.java @@ -30,6 +30,7 @@ import build.bazel.remote.execution.v2.Command; import build.bazel.remote.execution.v2.ContentAddressableStorageGrpc.ContentAddressableStorageImplBase; import build.bazel.remote.execution.v2.Digest; +import build.bazel.remote.execution.v2.DigestFunction; import build.bazel.remote.execution.v2.Directory; import build.bazel.remote.execution.v2.DirectoryNode; import build.bazel.remote.execution.v2.FileNode; @@ -799,6 +800,7 @@ public void updateActionResult( assertThat(request) .isEqualTo( UpdateActionResultRequest.newBuilder() + .setDigestFunction(DigestFunction.Value.SHA256) .setActionDigest(fooDigest) .setActionResult(result) .build()); From ee918697ea776e03552ed2457421eda75fc5229f Mon Sep 17 00:00:00 2001 From: "Ian (Hee) Cha" Date: Mon, 31 Jul 2023 10:52:45 -0700 Subject: [PATCH 010/125] Merge `use_repo` buildifier fixups into a single command (#19134) Users only have to copy&paste a single command this way. Closes #19119. PiperOrigin-RevId: 551887901 Change-Id: Id285563e83a18ac9a4db3b2d4e8d731d5b3f5813 Co-authored-by: Fabian Meumertzheim --- .../bazel/bzlmod/ModuleExtensionMetadata.java | 16 +++---- .../bzlmod/ModuleExtensionResolutionTest.java | 44 ++++++++----------- 2 files changed, 26 insertions(+), 34 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionMetadata.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionMetadata.java index f71986f509c8c9..a161cf0cff2e59 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionMetadata.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionMetadata.java @@ -16,6 +16,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static java.util.stream.Collectors.joining; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; @@ -31,7 +32,6 @@ import java.util.LinkedHashSet; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nullable; import net.starlark.java.annot.StarlarkBuiltin; @@ -285,7 +285,7 @@ private static Optional generateFixupMessage( String.join(", ", indirectDepImports)); } - var fixupCommands = + var fixupCommand = Stream.of( makeUseRepoCommand( "use_repo_add", @@ -315,19 +315,18 @@ private static Optional generateFixupMessage( extensionBzlFile, extensionName, rootUsage.getIsolationKey())) - .flatMap(Optional::stream); + .flatMap(Optional::stream) + .collect(joining(" ", "buildozer ", " //MODULE.bazel:all")); return Optional.of( Event.warn( location, message + String.format( - "%s ** You can use the following buildozer command(s) to fix these" + "%s ** You can use the following buildozer command to fix these" + " issues:%s\n\n" + "%s", - "\033[35m\033[1m", - "\033[0m", - fixupCommands.collect(Collectors.joining("\n"))))); + "\033[35m\033[1m", "\033[0m", fixupCommand))); } private static Optional makeUseRepoCommand( @@ -354,8 +353,7 @@ private static Optional makeUseRepoCommand( commandParts.add(extensionName); } commandParts.addAll(repos); - return Optional.of( - String.format("buildozer '%s' //MODULE.bazel:all", String.join(" ", commandParts))); + return Optional.of(commandParts.stream().collect(joining(" ", "'", "'"))); } private Optional> getRootModuleDirectDeps(Set allRepos) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java index 675a9ce4f7816c..5bd81dac41e5eb 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java @@ -1841,16 +1841,13 @@ public void extensionMetadata() throws Exception { + "Imported, but reported as indirect dependencies by the extension:\n" + " indirect_dep, indirect_dev_dep\n" + "\n" - + "\033[35m\033[1m ** You can use the following buildozer command(s) to fix these" + + "\033[35m\033[1m ** You can use the following buildozer command to fix these" + " issues:\033[0m\n" + "\n" + "buildozer 'use_repo_add @ext//:defs.bzl ext missing_direct_dep non_dev_as_dev_dep'" - + " //MODULE.bazel:all\n" - + "buildozer 'use_repo_remove @ext//:defs.bzl ext dev_as_non_dev_dep" - + " indirect_dep invalid_dep' //MODULE.bazel:all\n" - + "buildozer 'use_repo_add dev @ext//:defs.bzl ext dev_as_non_dev_dep" - + " missing_direct_dev_dep' //MODULE.bazel:all\n" - + "buildozer 'use_repo_remove dev @ext//:defs.bzl ext indirect_dev_dep invalid_dev_dep" + + " 'use_repo_remove @ext//:defs.bzl ext dev_as_non_dev_dep indirect_dep invalid_dep'" + + " 'use_repo_add dev @ext//:defs.bzl ext dev_as_non_dev_dep missing_direct_dev_dep'" + + " 'use_repo_remove dev @ext//:defs.bzl ext indirect_dev_dep invalid_dev_dep" + " non_dev_as_dev_dep' //MODULE.bazel:all", ImmutableSet.of(EventKind.WARNING)); } @@ -1925,14 +1922,13 @@ public void extensionMetadata_all() throws Exception { + " extension (may cause the build to fail when used by other modules):\n" + " direct_dev_dep, indirect_dev_dep\n" + "\n" - + "\033[35m\033[1m ** You can use the following buildozer command(s) to fix these" + + "\033[35m\033[1m ** You can use the following buildozer command to fix these" + " issues:\033[0m\n" + "\n" + "buildozer 'use_repo_add @ext//:defs.bzl ext direct_dev_dep indirect_dev_dep" - + " missing_direct_dep missing_direct_dev_dep' //MODULE.bazel:all\n" - + "buildozer 'use_repo_remove @ext//:defs.bzl ext invalid_dep' //MODULE.bazel:all\n" - + "buildozer 'use_repo_remove dev @ext//:defs.bzl ext direct_dev_dep indirect_dev_dep" - + " invalid_dev_dep' //MODULE.bazel:all", + + " missing_direct_dep missing_direct_dev_dep' 'use_repo_remove @ext//:defs.bzl ext" + + " invalid_dep' 'use_repo_remove dev @ext//:defs.bzl ext direct_dev_dep" + + " indirect_dev_dep invalid_dev_dep' //MODULE.bazel:all", ImmutableSet.of(EventKind.WARNING)); } @@ -2008,14 +2004,12 @@ public void extensionMetadata_allDev() throws Exception { + " extension (may cause the build to fail when used by other modules):\n" + " direct_dep, indirect_dep\n" + "\n" - + "\033[35m\033[1m ** You can use the following buildozer command(s) to fix these" + + "\033[35m\033[1m ** You can use the following buildozer command to fix these" + " issues:\033[0m\n" + "\n" + "buildozer 'use_repo_remove @ext//:defs.bzl ext direct_dep indirect_dep invalid_dep'" - + " //MODULE.bazel:all\n" - + "buildozer 'use_repo_add dev @ext//:defs.bzl ext direct_dep indirect_dep" - + " missing_direct_dep missing_direct_dev_dep' //MODULE.bazel:all\n" - + "buildozer 'use_repo_remove dev @ext//:defs.bzl ext invalid_dev_dep'" + + " 'use_repo_add dev @ext//:defs.bzl ext direct_dep indirect_dep missing_direct_dep" + + " missing_direct_dev_dep' 'use_repo_remove dev @ext//:defs.bzl ext invalid_dev_dep'" + " //MODULE.bazel:all", ImmutableSet.of(EventKind.WARNING)); } @@ -2129,11 +2123,11 @@ public void extensionMetadata_isolated() throws Exception { + "Imported, but reported as indirect dependencies by the extension:\n" + " indirect_dep\n" + "\n" - + "\033[35m\033[1m ** You can use the following buildozer command(s) to fix these" + + "\033[35m\033[1m ** You can use the following buildozer command to fix these" + " issues:\033[0m\n" + "\n" - + "buildozer 'use_repo_add ext1 direct_dep missing_direct_dep' //MODULE.bazel:all\n" - + "buildozer 'use_repo_remove ext1 indirect_dep' //MODULE.bazel:all", + + "buildozer 'use_repo_add ext1 direct_dep missing_direct_dep' 'use_repo_remove ext1" + + " indirect_dep' //MODULE.bazel:all", ImmutableSet.of(EventKind.WARNING)); assertContainsEvent( "WARNING /ws/MODULE.bazel:8:21: The module extension ext defined in @ext//:defs.bzl" @@ -2143,7 +2137,7 @@ public void extensionMetadata_isolated() throws Exception { + " build to fail):\n" + " missing_direct_dep\n" + "\n" - + "\033[35m\033[1m ** You can use the following buildozer command(s) to fix these" + + "\033[35m\033[1m ** You can use the following buildozer command to fix these" + " issues:\033[0m\n" + "\n" + "buildozer 'use_repo_add ext2 missing_direct_dep' //MODULE.bazel:all", @@ -2213,11 +2207,11 @@ public void extensionMetadata_isolatedDev() throws Exception { + "Imported, but reported as indirect dependencies by the extension:\n" + " indirect_dep\n" + "\n" - + "\033[35m\033[1m ** You can use the following buildozer command(s) to fix these" + + "\033[35m\033[1m ** You can use the following buildozer command to fix these" + " issues:\033[0m\n" + "\n" - + "buildozer 'use_repo_add ext1 direct_dep missing_direct_dep' //MODULE.bazel:all\n" - + "buildozer 'use_repo_remove ext1 indirect_dep' //MODULE.bazel:all", + + "buildozer 'use_repo_add ext1 direct_dep missing_direct_dep' 'use_repo_remove ext1" + + " indirect_dep' //MODULE.bazel:all", ImmutableSet.of(EventKind.WARNING)); assertContainsEvent( "WARNING /ws/MODULE.bazel:8:21: The module extension ext defined in @ext//:defs.bzl" @@ -2227,7 +2221,7 @@ public void extensionMetadata_isolatedDev() throws Exception { + " build to fail):\n" + " missing_direct_dep\n" + "\n" - + "\033[35m\033[1m ** You can use the following buildozer command(s) to fix these" + + "\033[35m\033[1m ** You can use the following buildozer command to fix these" + " issues:\033[0m\n" + "\n" + "buildozer 'use_repo_add ext2 missing_direct_dep' //MODULE.bazel:all", From 382dfafa1a77e0eaabe6aa11e92176323e675499 Mon Sep 17 00:00:00 2001 From: "Ian (Hee) Cha" Date: Thu, 3 Aug 2023 14:10:10 -0700 Subject: [PATCH 011/125] Ensure that extension unique names followed by `~` are prefix-free (#19164) Fixes errors such as the following when a module provides two extensions that share a name: ``` ERROR: /my/workspace/library_path/BUILD:4:13: no such package '@_main~my_ext~2~xyz//': The repository '@_main~my_ext~2~xyz' could not be resolved: Repository '@_main~my_ext~2~xyz' is not defined and referenced by '//library_path:library_name' ERROR: Analysis of target '//my_path:my_target' failed; build aborted: ``` This was a consequence of the extension unique names followed by `~` being ``` _main~my_ext~2~ _main~my_ext~ ``` since 19a971089952e53bba130a8e3e8b28b6b1e310e6. Before, they were of the following form, which was prefix-free: ``` _main~my_ext2~ _main~my_ext~ ``` Fixes #19155 Closes #19156. PiperOrigin-RevId: 553567725 Change-Id: I98650663fea3bfee77752a06a99132e507d91aef Co-authored-by: Fabian Meumertzheim --- .../bazel/bzlmod/BazelDepGraphFunction.java | 59 +++++++++++-------- .../bzlmod/BazelDepGraphFunctionTest.java | 4 +- .../bzlmod/ModuleExtensionResolutionTest.java | 49 +++++++++++++++ 3 files changed, 84 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java index eaf358a011d05b..9e7c0a328e8356 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java @@ -20,6 +20,7 @@ import static com.google.common.collect.ImmutableMap.toImmutableMap; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableBiMap; @@ -247,39 +248,45 @@ static BzlmodFlagsAndEnvVars getFlagsAndEnvVars(Environment env) throws Interrup private ImmutableBiMap calculateUniqueNameForUsedExtensionId( ImmutableTable extensionUsagesById) { - // Calculate a unique name for each used extension id. + // Calculate a unique name for each used extension id with the following property that is + // required for BzlmodRepoRuleFunction to unambiguously identify the extension that generates a + // given repo: + // After appending a single `~` to each such name, none of the resulting strings is a prefix of + // any other such string. BiMap extensionUniqueNames = HashBiMap.create(); for (ModuleExtensionId id : extensionUsagesById.rowKeySet()) { - // Ensure that the resulting extension name (and thus the repository names derived from it) do - // not start with a tilde. - RepositoryName repository = id.getBzlFileLabel().getRepository(); - String nonEmptyRepoPart = repository.isMain() ? "_main" : repository.getName(); - // When using a namespace, prefix the extension name with "_" to distinguish the prefix from - // those generated by non-namespaced extension usages. Extension names are identified by their - // Starlark identifier, which in the case of an exported symbol cannot start with "_". - String bestName = - id.getIsolationKey() - .map( - namespace -> - String.format( - "%s~_%s~%s~%s~%s", - nonEmptyRepoPart, - id.getExtensionName(), - namespace.getModule().getName(), - namespace.getModule().getVersion(), - namespace.getUsageExportedName())) - .orElse(nonEmptyRepoPart + "~" + id.getExtensionName()); - if (extensionUniqueNames.putIfAbsent(bestName, id) == null) { - continue; - } - int suffix = 2; - while (extensionUniqueNames.putIfAbsent(bestName + "~" + suffix, id) != null) { - suffix++; + int attempt = 1; + while (extensionUniqueNames.putIfAbsent(makeUniqueNameCandidate(id, attempt), id) != null) { + attempt++; } } return ImmutableBiMap.copyOf(extensionUniqueNames); } + private static String makeUniqueNameCandidate(ModuleExtensionId id, int attempt) { + // Ensure that the resulting extension name (and thus the repository names derived from it) do + // not start with a tilde. + RepositoryName repository = id.getBzlFileLabel().getRepository(); + String nonEmptyRepoPart = repository.isMain() ? "_main" : repository.getName(); + // When using a namespace, prefix the extension name with "_" to distinguish the prefix from + // those generated by non-namespaced extension usages. Extension names are identified by their + // Starlark identifier, which in the case of an exported symbol cannot start with "_". + Preconditions.checkArgument(attempt >= 1); + String extensionNameDisambiguator = attempt == 1 ? "" : String.valueOf(attempt); + return id.getIsolationKey() + .map( + namespace -> + String.format( + "%s~_%s%s~%s~%s~%s", + nonEmptyRepoPart, + id.getExtensionName(), + extensionNameDisambiguator, + namespace.getModule().getName(), + namespace.getModule().getVersion(), + namespace.getUsageExportedName())) + .orElse(nonEmptyRepoPart + "~" + id.getExtensionName() + extensionNameDisambiguator); + } + static class BazelDepGraphFunctionException extends SkyFunctionException { BazelDepGraphFunctionException(ExternalDepsException e, Transience transience) { super(e, transience); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java index fbe92cf779386f..b8602dba90d16e 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java @@ -292,7 +292,7 @@ public void createValue_moduleExtensions() throws Exception { maven, "rules_jvm_external~1.0~maven", pip, "rules_python~2.0~pip", myext, "dep~2.0~myext", - myext2, "dep~2.0~myext~2"); + myext2, "dep~2.0~myext2"); assertThat(value.getFullRepoMapping(ModuleKey.ROOT)) .isEqualTo( @@ -323,7 +323,7 @@ public void createValue_moduleExtensions() throws Exception { "oneext", "dep~2.0~myext~myext", "twoext", - "dep~2.0~myext~2~myext")); + "dep~2.0~myext2~myext")); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java index 5bd81dac41e5eb..e1e10dfbd92069 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java @@ -419,6 +419,55 @@ public void simpleExtension_nonCanonicalLabel_repoName() throws Exception { assertThat(result.get(skyKey).getModule().getGlobal("data")).isEqualTo("foo:fu bar:ba quz:qu"); } + @Test + public void multipleExtensions_sameName() throws Exception { + scratch.file( + workspaceRoot.getRelative("MODULE.bazel").getPathString(), + "bazel_dep(name='data_repo', version='1.0')", + "first_ext = use_extension('//first_ext:defs.bzl', 'ext')", + "first_ext.tag(name='foo', data='first_fu')", + "first_ext.tag(name='bar', data='first_ba')", + "use_repo(first_ext, first_foo='foo', first_bar='bar')", + "second_ext = use_extension('//second_ext:defs.bzl', 'ext')", + "second_ext.tag(name='foo', data='second_fu')", + "second_ext.tag(name='bar', data='second_ba')", + "use_repo(second_ext, second_foo='foo', second_bar='bar')"); + scratch.file(workspaceRoot.getRelative("first_ext/BUILD").getPathString()); + scratch.file( + workspaceRoot.getRelative("first_ext/defs.bzl").getPathString(), + "load('@data_repo//:defs.bzl','data_repo')", + "tag = tag_class(attrs = {'name':attr.string(),'data':attr.string()})", + "def _ext_impl(ctx):", + " for mod in ctx.modules:", + " for tag in mod.tags.tag:", + " data_repo(name=tag.name,data=tag.data)", + "ext = module_extension(implementation=_ext_impl, tag_classes={'tag':tag})"); + scratch.file(workspaceRoot.getRelative("second_ext/BUILD").getPathString()); + scratch.file( + workspaceRoot.getRelative("second_ext/defs.bzl").getPathString(), + "load('//first_ext:defs.bzl', _ext = 'ext')", + "ext = _ext"); + scratch.file(workspaceRoot.getRelative("BUILD").getPathString()); + scratch.file( + workspaceRoot.getRelative("data.bzl").getPathString(), + "load('@first_foo//:data.bzl', first_foo_data='data')", + "load('@first_bar//:data.bzl', first_bar_data='data')", + "load('@second_foo//:data.bzl', second_foo_data='data')", + "load('@second_bar//:data.bzl', second_bar_data='data')", + "data = 'first_foo:'+first_foo_data+' first_bar:'+first_bar_data" + + "+' second_foo:'+second_foo_data+' second_bar:'+second_bar_data"); + + SkyKey skyKey = BzlLoadValue.keyForBuild(Label.parseCanonical("//:data.bzl")); + EvaluationResult result = + evaluator.evaluate(ImmutableList.of(skyKey), evaluationContext); + if (result.hasError()) { + throw result.getError().getException(); + } + assertThat(result.get(skyKey).getModule().getGlobal("data")) + .isEqualTo( + "first_foo:first_fu first_bar:first_ba second_foo:second_fu " + "second_bar:second_ba"); + } + @Test public void multipleModules() throws Exception { scratch.file( From 1bc0f87fcd05c655b6f1ed0ffcabad47e8026baa Mon Sep 17 00:00:00 2001 From: Salma Samy Date: Fri, 4 Aug 2023 03:17:32 +0300 Subject: [PATCH 012/125] [6.4.0] Lockfile updates & fixes (#19153) * Add 'environ' attribute to module extensions and update lockfile - Add 'environ' to module extensions - Store the variables and their values in the lockfile - Compare the variables and values with the lockfile and re-evaluate or error if they differ PiperOrigin-RevId: 548964193 Change-Id: Ic2d52fe3332e93095c414d8bca4c9b4312bca8c2 # Conflicts: # src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtension.java # src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java # src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryModule.java # src/main/java/com/google/devtools/build/lib/skyframe/ActionEnvironmentFunction.java # src/main/java/com/google/devtools/build/lib/skyframe/ClientEnvironmentFunction.java # src/main/java/com/google/devtools/build/lib/starlarkbuildapi/repository/RepositoryModuleApi.java # src/main/java/com/google/devtools/build/skydoc/fakebuildapi/repository/FakeRepositoryModule.java # src/test/java/com/google/devtools/build/lib/bazel/bzlmod/StarlarkBazelModuleTest.java # src/test/py/bazel/bzlmod/bazel_lockfile_test.py * Remove extra changes * Re-run extension if its files change fixes https://github.com/bazelbuild/bazel/issues/19068 PiperOrigin-RevId: 552823534 Change-Id: I87256b2cf954b932e24c70e22386020599f21a6f * Do not fail on new fields added to lockfile data Fixes https://github.com/bazelbuild/bazel/issues/19105 PiperOrigin-RevId: 553068023 Change-Id: I877bc8ece0641c01119a9295e09175a2d0a3a0c1 --------- Co-authored-by: Ian (Hee) Cha --- .../lib/bazel/bzlmod/BazelLockFileValue.java | 31 ++- .../lib/bazel/bzlmod/GsonTypeAdapterUtil.java | 14 + .../bazel/bzlmod/LockFileModuleExtension.java | 30 ++- .../lib/bazel/bzlmod/ModuleExtension.java | 8 +- .../bzlmod/SingleExtensionEvalFunction.java | 139 ++++++++-- .../starlark/StarlarkRepositoryModule.java | 11 + .../rules/repository/RepositoryFunction.java | 52 ++-- .../skyframe/ActionEnvironmentFunction.java | 19 +- .../skyframe/ClientEnvironmentFunction.java | 3 +- .../repository/FakeRepositoryModule.java | 3 +- .../bazel/bzlmod/StarlarkBazelModuleTest.java | 2 + .../py/bazel/bzlmod/bazel_lockfile_test.py | 249 +++++++++++++----- 12 files changed, 428 insertions(+), 133 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java index c0d0c06aed41e4..26c674fb6bc14f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java @@ -109,25 +109,34 @@ public ImmutableList getModuleAndFlagsDiff( return moduleDiff.build(); } + /** Returns the differences between an extension and its locked data */ public ImmutableList getModuleExtensionDiff( - LockFileModuleExtension lockedExtension, - ImmutableMap lockedExtensionUsages, ModuleExtensionId extensionId, byte[] transitiveDigest, - ImmutableMap extensionUsages) { + boolean filesChanged, + ImmutableMap envVariables, + ImmutableMap extensionUsages, + ImmutableMap lockedExtensionUsages) { + LockFileModuleExtension lockedExtension = getModuleExtensions().get(extensionId); + ImmutableList.Builder extDiff = new ImmutableList.Builder<>(); - if (lockedExtension == null) { - extDiff.add("The module extension '" + extensionId + "' does not exist in the lockfile"); - } else { - if (!Arrays.equals(transitiveDigest, lockedExtension.getBzlTransitiveDigest())) { + if (!Arrays.equals(transitiveDigest, lockedExtension.getBzlTransitiveDigest())) { extDiff.add( "The implementation of the extension '" + extensionId + "' or one of its transitive .bzl files has changed"); - } - if (!extensionUsages.equals(lockedExtensionUsages)) { - extDiff.add("The usages of the extension named '" + extensionId + "' has changed"); - } + } + if (filesChanged) { + extDiff.add("One or more files the extension '" + extensionId + "' is using have changed"); + } + if (!extensionUsages.equals(lockedExtensionUsages)) { + extDiff.add("The usages of the extension '" + extensionId + "' has changed"); + } + if (!envVariables.equals(lockedExtension.getEnvVariables())) { + extDiff.add( + "The environment variables the extension '" + + extensionId + + "' depends on (or their values) have changed"); } return extDiff.build(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java index f0190f4c9fb66f..334484716f8bc2 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java @@ -91,6 +91,19 @@ public ModuleKey read(JsonReader jsonReader) throws IOException { } }; + public static final TypeAdapter

*/ .add(attr("linkopts", STRING_LIST)) + /* + Pass these files to the C++ linker command. +

+ For example, compiled Windows .res files can be provided here to be embedded in + the binary target. +

+ */ + .add( + attr("additional_linker_inputs", LABEL_LIST) + .orderIndependent() + .direct_compile_time_input() + .allowedFileTypes(FileTypeSet.ANY_FILE)) /* Remove matching options from the C++ compilation command. Subject to "Make" variable substitution. @@ -496,18 +508,6 @@ public CcBinaryBaseRule(GraphNodeAspect graphNodeAspect) { @Override public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) { return builder - /* - Pass these files to the C++ linker command. -

- For example, compiled Windows .res files can be provided here to be embedded in - the binary target. -

- */ - .add( - attr("additional_linker_inputs", LABEL_LIST) - .orderIndependent() - .direct_compile_time_input() - .allowedFileTypes(FileTypeSet.ANY_FILE)) .override( attr("deps", LABEL_LIST) .allowedRuleClasses(DEPS_ALLOWED_RULES) diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_library.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_library.bzl index 45ad0c8925d36f..d82642cd9cf3e0 100755 --- a/src/main/starlark/builtins_bzl/common/cc/cc_library.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/cc_library.bzl @@ -151,7 +151,7 @@ def _cc_library_impl(ctx): compilation_outputs = compilation_outputs, cc_toolchain = cc_toolchain, feature_configuration = feature_configuration, - additional_inputs = _filter_linker_scripts(ctx.files.deps), + additional_inputs = _filter_linker_scripts(ctx.files.deps) + ctx.files.additional_linker_inputs, linking_contexts = linking_contexts, grep_includes = ctx.executable._grep_includes, user_link_flags = common.linkopts, @@ -209,11 +209,12 @@ def _cc_library_impl(ctx): else: user_link_flags = common.linkopts linker_scripts = _filter_linker_scripts(ctx.files.deps) - if len(common.linkopts) > 0 or len(linker_scripts) > 0 or not semantics.should_create_empty_archive(): + additional_linker_inputs = ctx.files.additional_linker_inputs + if len(user_link_flags) > 0 or len(linker_scripts) > 0 or len(additional_linker_inputs) > 0 or not semantics.should_create_empty_archive(): linker_input = cc_common.create_linker_input( owner = ctx.label, user_link_flags = common.linkopts, - additional_inputs = depset(linker_scripts), + additional_inputs = depset(linker_scripts + additional_linker_inputs), ) contexts_to_merge.append(cc_common.create_linking_context(linker_inputs = depset([linker_input]))) @@ -583,6 +584,10 @@ attrs = { "linkstamp": attr.label(allow_single_file = True), "linkopts": attr.string_list(), "nocopts": attr.string(), + "additional_linker_inputs": attr.label_list( + allow_files = True, + flags = ["ORDER_INDEPENDENT", "DIRECT_COMPILE_TIME_INPUT"], + ), "includes": attr.string_list(), "defines": attr.string_list(), "copts": attr.string_list(), From 333e83f648795669695821c1c1ce77eb9e7b01c1 Mon Sep 17 00:00:00 2001 From: "Ian (Hee) Cha" Date: Tue, 22 Aug 2023 12:34:29 -0700 Subject: [PATCH 033/125] [6.4.0] Do not rerun module extensions when only imports or locations change (#19284) When using the lockfile, extension evaluation results are now persisted even if the imports (`use_repo`s) fail validation against the actually generated repos and reused if only imports or Starlark locations change. This requires storing `ModuleExtensionMetadata` in the lockfile to ensure that the correct fixup warnings are shown if imports are updated but the extension isn't rerun. Closes #19253. Commit https://github.com/bazelbuild/bazel/commit/2c2b07bb23e3b4028961fe2fd6538010436b171e PiperOrigin-RevId: 557878432 Change-Id: I25909294b118fb445f9c48f61e18463762f78208 Co-authored-by: Fabian Meumertzheim --- .../devtools/build/lib/bazel/bzlmod/BUILD | 24 ++- .../lib/bazel/bzlmod/BazelLockFileValue.java | 2 +- .../bazel/bzlmod/LockFileModuleExtension.java | 8 +- .../bazel/bzlmod/ModuleExtensionMetadata.java | 55 ++++--- .../bazel/bzlmod/ModuleExtensionUsage.java | 2 + .../bzlmod/SingleExtensionEvalFunction.java | 149 ++++++++++++++---- .../py/bazel/bzlmod/bazel_lockfile_test.py | 125 +++++++++++++++ 7 files changed, 304 insertions(+), 61 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 31ed023aeb27c7..60e7f592e856bd 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -138,6 +138,7 @@ java_library( ":common", ":inspection", ":module_extension", + ":module_extension_metadata", ":registry", ":repo_rule_creator", "//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories", @@ -170,7 +171,6 @@ java_library( "Discovery.java", "GsonTypeAdapterUtil.java", "ModuleExtensionContext.java", - "ModuleExtensionMetadata.java", "ModuleFileFunction.java", "ModuleFileGlobals.java", "Selection.java", @@ -184,6 +184,7 @@ java_library( ":common", ":exception", ":module_extension", + ":module_extension_metadata", ":registry", ":resolution", "//src/main/java/com/google/devtools/build/docgen/annot", @@ -284,3 +285,24 @@ java_library( "//third_party:jsr305", ], ) + +java_library( + name = "module_extension_metadata", + srcs = [ + "ModuleExtensionMetadata.java", + ], + deps = [ + ":common", + ":module_extension", + "//src/main/java/com/google/devtools/build/docgen/annot", + "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/events", + "//src/main/java/net/starlark/java/annot", + "//src/main/java/net/starlark/java/eval", + "//src/main/java/net/starlark/java/syntax", + "//third_party:auto_value", + "//third_party:gson", + "//third_party:guava", + "//third_party:jsr305", + ], +) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java index 26c674fb6bc14f..af422048956416 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java @@ -130,7 +130,7 @@ public ImmutableList getModuleExtensionDiff( extDiff.add("One or more files the extension '" + extensionId + "' is using have changed"); } if (!extensionUsages.equals(lockedExtensionUsages)) { - extDiff.add("The usages of the extension '" + extensionId + "' has changed"); + extDiff.add("The usages of the extension '" + extensionId + "' have changed"); } if (!envVariables.equals(lockedExtension.getEnvVariables())) { extDiff.add( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java index 1b5eb71d60c35b..9b111974694c19 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/LockFileModuleExtension.java @@ -19,6 +19,7 @@ import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; import com.ryanharter.auto.value.gson.GenerateTypeAdapter; +import java.util.Optional; /** * This object serves as a container for the transitive digest (obtained from transitive .bzl files) @@ -33,7 +34,8 @@ public static Builder builder() { return new AutoValue_LockFileModuleExtension.Builder() // TODO(salmasamy) can be removed when updating lockfile version .setEnvVariables(ImmutableMap.of()) - .setAccumulatedFileDigests(ImmutableMap.of()); + .setAccumulatedFileDigests(ImmutableMap.of()) + .setModuleExtensionMetadata(Optional.empty()); } @SuppressWarnings("mutable") @@ -45,6 +47,8 @@ public static Builder builder() { public abstract ImmutableMap getGeneratedRepoSpecs(); + public abstract Optional getModuleExtensionMetadata(); + public abstract Builder toBuilder(); /** Builder type for {@link LockFileModuleExtension}. */ @@ -59,6 +63,8 @@ public abstract static class Builder { public abstract Builder setGeneratedRepoSpecs(ImmutableMap value); + public abstract Builder setModuleExtensionMetadata(Optional value); + public abstract LockFileModuleExtension build(); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionMetadata.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionMetadata.java index a161cf0cff2e59..443a71c86b69db 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionMetadata.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionMetadata.java @@ -18,6 +18,7 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet; import static java.util.stream.Collectors.joining; +import com.google.auto.value.AutoValue; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedSet; @@ -27,6 +28,7 @@ import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; +import com.ryanharter.auto.value.gson.GenerateTypeAdapter; import java.util.ArrayList; import java.util.Collection; import java.util.LinkedHashSet; @@ -49,24 +51,29 @@ doc = "Return values of this type from a module extension's implementation function to " + "provide metadata about the repositories generated by the extension to Bazel.") -public class ModuleExtensionMetadata implements StarlarkValue { - @Nullable private final ImmutableSet explicitRootModuleDirectDeps; - @Nullable private final ImmutableSet explicitRootModuleDirectDevDeps; - private final UseAllRepos useAllRepos; +@AutoValue +@GenerateTypeAdapter +public abstract class ModuleExtensionMetadata implements StarlarkValue { + @Nullable + abstract ImmutableSet getExplicitRootModuleDirectDeps(); - private ModuleExtensionMetadata( + @Nullable + abstract ImmutableSet getExplicitRootModuleDirectDevDeps(); + + abstract UseAllRepos getUseAllRepos(); + + private static ModuleExtensionMetadata create( @Nullable Set explicitRootModuleDirectDeps, @Nullable Set explicitRootModuleDirectDevDeps, UseAllRepos useAllRepos) { - this.explicitRootModuleDirectDeps = + return new AutoValue_ModuleExtensionMetadata( explicitRootModuleDirectDeps != null ? ImmutableSet.copyOf(explicitRootModuleDirectDeps) - : null; - this.explicitRootModuleDirectDevDeps = + : null, explicitRootModuleDirectDevDeps != null ? ImmutableSet.copyOf(explicitRootModuleDirectDevDeps) - : null; - this.useAllRepos = useAllRepos; + : null, + useAllRepos); } static ModuleExtensionMetadata create( @@ -76,19 +83,19 @@ static ModuleExtensionMetadata create( throws EvalException { if (rootModuleDirectDepsUnchecked == Starlark.NONE && rootModuleDirectDevDepsUnchecked == Starlark.NONE) { - return new ModuleExtensionMetadata(null, null, UseAllRepos.NO); + return create(null, null, UseAllRepos.NO); } // When root_module_direct_deps = "all", accept both root_module_direct_dev_deps = None and // root_module_direct_dev_deps = [], but not root_module_direct_dev_deps = ["some_repo"]. if (rootModuleDirectDepsUnchecked.equals("all") && rootModuleDirectDevDepsUnchecked.equals(StarlarkList.immutableOf())) { - return new ModuleExtensionMetadata(null, null, UseAllRepos.REGULAR); + return create(null, null, UseAllRepos.REGULAR); } if (rootModuleDirectDevDepsUnchecked.equals("all") && rootModuleDirectDepsUnchecked.equals(StarlarkList.immutableOf())) { - return new ModuleExtensionMetadata(null, null, UseAllRepos.DEV); + return create(null, null, UseAllRepos.DEV); } if (rootModuleDirectDepsUnchecked.equals("all") @@ -146,8 +153,7 @@ static ModuleExtensionMetadata create( } } - return new ModuleExtensionMetadata( - explicitRootModuleDirectDeps, explicitRootModuleDirectDevDeps, UseAllRepos.NO); + return create(explicitRootModuleDirectDeps, explicitRootModuleDirectDevDeps, UseAllRepos.NO); } public void evaluate( @@ -358,10 +364,10 @@ private static Optional makeUseRepoCommand( private Optional> getRootModuleDirectDeps(Set allRepos) throws EvalException { - switch (useAllRepos) { + switch (getUseAllRepos()) { case NO: - if (explicitRootModuleDirectDeps != null) { - Set invalidRepos = Sets.difference(explicitRootModuleDirectDeps, allRepos); + if (getExplicitRootModuleDirectDeps() != null) { + Set invalidRepos = Sets.difference(getExplicitRootModuleDirectDeps(), allRepos); if (!invalidRepos.isEmpty()) { throw Starlark.errorf( "root_module_direct_deps contained the following repositories " @@ -369,7 +375,7 @@ private Optional> getRootModuleDirectDeps(Set allRe String.join(", ", invalidRepos)); } } - return Optional.ofNullable(explicitRootModuleDirectDeps); + return Optional.ofNullable(getExplicitRootModuleDirectDeps()); case REGULAR: return Optional.of(ImmutableSet.copyOf(allRepos)); case DEV: @@ -380,10 +386,11 @@ private Optional> getRootModuleDirectDeps(Set allRe private Optional> getRootModuleDirectDevDeps(Set allRepos) throws EvalException { - switch (useAllRepos) { + switch (getUseAllRepos()) { case NO: - if (explicitRootModuleDirectDevDeps != null) { - Set invalidRepos = Sets.difference(explicitRootModuleDirectDevDeps, allRepos); + if (getExplicitRootModuleDirectDevDeps() != null) { + Set invalidRepos = + Sets.difference(getExplicitRootModuleDirectDevDeps(), allRepos); if (!invalidRepos.isEmpty()) { throw Starlark.errorf( "root_module_direct_dev_deps contained the following " @@ -391,7 +398,7 @@ private Optional> getRootModuleDirectDevDeps(Set al String.join(", ", invalidRepos)); } } - return Optional.ofNullable(explicitRootModuleDirectDevDeps); + return Optional.ofNullable(getExplicitRootModuleDirectDevDeps()); case REGULAR: return Optional.of(ImmutableSet.of()); case DEV: @@ -400,7 +407,7 @@ private Optional> getRootModuleDirectDevDeps(Set al throw new IllegalStateException("not reached"); } - private enum UseAllRepos { + enum UseAllRepos { NO, REGULAR, DEV, diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionUsage.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionUsage.java index c84f87d62529f2..9d186432afef10 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionUsage.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionUsage.java @@ -80,6 +80,8 @@ public abstract class ModuleExtensionUsage { */ public abstract boolean getHasNonDevUseExtension(); + public abstract Builder toBuilder(); + public static Builder builder() { return new AutoValue_ModuleExtensionUsage.Builder(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java index 4d72d021d9541e..8dbdadac7333ff 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java @@ -19,10 +19,12 @@ import static com.google.common.collect.ImmutableSet.toImmutableSet; import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableTable; +import com.google.common.collect.Maps; import com.google.devtools.build.lib.actions.FileValue; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; @@ -57,6 +59,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.function.Function; import java.util.function.Supplier; import javax.annotation.Nullable; @@ -190,9 +193,14 @@ public SkyValue compute(SkyKey skyKey, Environment env) } ImmutableMap generatedRepoSpecs = moduleExtensionResult.getGeneratedRepoSpecs(); - // Check that all imported repos have been actually generated - validateAllImportsAreGenerated(generatedRepoSpecs, usagesValue, extensionId); - + Optional moduleExtensionMetadata = + moduleExtensionResult.getModuleExtensionMetadata(); + + // At this point the extension has been evaluated successfully, but SingleExtensionEvalFunction + // may still fail if imported repositories were not generated. However, since imports do not + // influence the evaluation of the extension and the validation also runs when the extension + // result is taken from the lockfile, we can already post the update event. This is necessary to + // prevent the extension from rerunning when only the imports change. if (lockfileMode.equals(LockfileMode.UPDATE)) { env.getListener() .post( @@ -203,9 +211,11 @@ public SkyValue compute(SkyKey skyKey, Environment env) .setAccumulatedFileDigests(moduleExtensionResult.getAccumulatedFileDigests()) .setEnvVariables(extensionEnvVars) .setGeneratedRepoSpecs(generatedRepoSpecs) + .setModuleExtensionMetadata(moduleExtensionMetadata) .build())); } - return createSingleExtentionValue(generatedRepoSpecs, usagesValue); + return validateAndCreateSingleExtensionEvalValue( + generatedRepoSpecs, moduleExtensionMetadata, extensionId, usagesValue, env); } @Nullable @@ -247,12 +257,22 @@ private SingleExtensionEvalValue tryGettingValueFromLockFile( return null; } - // Check extension data in lockfile still valid + // Check extension data in lockfile is still valid, disregarding usage information that is not + // relevant for the evaluation of the extension. + ImmutableMap trimmedLockedUsages = + trimUsagesForEvaluation(lockedExtensionUsages); + ImmutableMap trimmedUsages = + trimUsagesForEvaluation(usagesValue.getExtensionUsages()); if (!filesChanged && Arrays.equals(bzlTransitiveDigest, lockedExtension.getBzlTransitiveDigest()) - && usagesValue.getExtensionUsages().equals(lockedExtensionUsages) + && trimmedUsages.equals(trimmedLockedUsages) && envVariables.equals(lockedExtension.getEnvVariables())) { - return createSingleExtentionValue(lockedExtension.getGeneratedRepoSpecs(), usagesValue); + return validateAndCreateSingleExtensionEvalValue( + lockedExtension.getGeneratedRepoSpecs(), + lockedExtension.getModuleExtensionMetadata(), + extensionId, + usagesValue, + env); } else if (lockfileMode.equals(LockfileMode.ERROR)) { ImmutableList extDiff = lockfile.getModuleExtensionDiff( @@ -260,8 +280,8 @@ private SingleExtensionEvalValue tryGettingValueFromLockFile( bzlTransitiveDigest, filesChanged, envVariables, - usagesValue.getExtensionUsages(), - lockedExtensionUsages); + trimmedUsages, + trimmedLockedUsages); throw new SingleExtensionEvalFunctionException( ExternalDepsException.withMessage( Code.BAD_MODULE, @@ -309,24 +329,46 @@ private Boolean didFilesChange( return false; } - private SingleExtensionEvalValue createSingleExtentionValue( - ImmutableMap generatedRepoSpecs, SingleExtensionUsagesValue usagesValue) { - return SingleExtensionEvalValue.create( - generatedRepoSpecs, - generatedRepoSpecs.keySet().stream() - .collect( - toImmutableBiMap( - e -> - RepositoryName.createUnvalidated( - usagesValue.getExtensionUniqueName() + "~" + e), - Function.identity()))); - } - - private void validateAllImportsAreGenerated( + /** + * Validates the result of the module extension evaluation against the declared imports, throwing + * an exception if validation fails, and returns a SingleExtensionEvalValue otherwise. + * + *

Since extension evaluation does not depend on the declared imports, the result of the + * evaluation of the extension implementation function can be reused and persisted in the lockfile + * even if validation fails. + */ + private SingleExtensionEvalValue validateAndCreateSingleExtensionEvalValue( ImmutableMap generatedRepoSpecs, + Optional moduleExtensionMetadata, + ModuleExtensionId extensionId, SingleExtensionUsagesValue usagesValue, - ModuleExtensionId extensionId) + Environment env) throws SingleExtensionEvalFunctionException { + // Evaluate the metadata before failing on invalid imports so that fixup warning are still + // emitted in case of an error. + if (moduleExtensionMetadata.isPresent()) { + try { + // TODO: ModuleExtensionMetadata#evaluate should throw ExternalDepsException instead of + // EvalException. + moduleExtensionMetadata + .get() + .evaluate( + usagesValue.getExtensionUsages().values(), + generatedRepoSpecs.keySet(), + env.getListener()); + } catch (EvalException e) { + env.getListener().handle(Event.error(e.getMessageWithStack())); + throw new SingleExtensionEvalFunctionException( + ExternalDepsException.withMessage( + Code.BAD_MODULE, + "error evaluating module extension %s in %s", + extensionId.getExtensionName(), + extensionId.getBzlFileLabel()), + Transience.TRANSIENT); + } + } + + // Check that all imported repos have actually been generated. for (ModuleExtensionUsage usage : usagesValue.getExtensionUsages().values()) { for (Entry repoImport : usage.getImports().entrySet()) { if (!generatedRepoSpecs.containsKey(repoImport.getValue())) { @@ -345,6 +387,43 @@ private void validateAllImportsAreGenerated( } } } + + return SingleExtensionEvalValue.create( + generatedRepoSpecs, + generatedRepoSpecs.keySet().stream() + .collect( + toImmutableBiMap( + e -> + RepositoryName.createUnvalidated( + usagesValue.getExtensionUniqueName() + "~" + e), + Function.identity()))); + } + + /** + * Returns usages with all information removed that does not influence the evaluation of the + * extension. + */ + private static ImmutableMap trimUsagesForEvaluation( + Map usages) { + return ImmutableMap.copyOf( + Maps.transformValues( + usages, + usage -> + // We start with the full usage and selectively remove information that does not + // influence the evaluation of the extension. Compared to explicitly copying over + // the parts that do, this preserves correctness in case new fields are added to + // ModuleExtensionUsage without updating this code. + usage.toBuilder() + // Locations are only used for error reporting and thus don't influence whether + // the evaluation of the extension is successful and what its result is + // in case of success. + .setLocation(Location.BUILTIN) + // Extension implementation functions do not see the imports, they are only + // validated against the set of generated repos in a validation step that comes + // afterward. + .setImports(ImmutableBiMap.of()) + .setDevImports(ImmutableSet.of()) + .build())); } private BzlLoadValue loadBzlFile( @@ -401,6 +480,7 @@ private RunModuleExtensionResult runModuleExtension( directories, env.getListener()); ModuleExtensionContext moduleContext; + Optional moduleExtensionMetadata; try (Mutability mu = Mutability.create("module extension", usagesValue.getExtensionUniqueName())) { StarlarkThread thread = new StarlarkThread(mu, starlarkSemantics); @@ -422,11 +502,9 @@ private RunModuleExtensionResult runModuleExtension( Transience.PERSISTENT); } if (returnValue instanceof ModuleExtensionMetadata) { - ModuleExtensionMetadata metadata = (ModuleExtensionMetadata) returnValue; - metadata.evaluate( - usagesValue.getExtensionUsages().values(), - threadContext.getGeneratedRepoSpecs().keySet(), - env.getListener()); + moduleExtensionMetadata = Optional.of((ModuleExtensionMetadata) returnValue); + } else { + moduleExtensionMetadata = Optional.empty(); } } catch (NeedsSkyframeRestartException e) { // Clean up and restart by returning null. @@ -456,7 +534,9 @@ private RunModuleExtensionResult runModuleExtension( } } return RunModuleExtensionResult.create( - moduleContext.getAccumulatedFileDigests(), threadContext.getGeneratedRepoSpecs()); + moduleContext.getAccumulatedFileDigests(), + threadContext.getGeneratedRepoSpecs(), + moduleExtensionMetadata); } private ModuleExtensionContext createContext( @@ -517,13 +597,14 @@ abstract static class RunModuleExtensionResult { abstract ImmutableMap getGeneratedRepoSpecs(); + abstract Optional getModuleExtensionMetadata(); + static RunModuleExtensionResult create( ImmutableMap accumulatedFileDigests, - ImmutableMap generatedRepoSpecs) { + ImmutableMap generatedRepoSpecs, + Optional moduleExtensionMetadata) { return new AutoValue_SingleExtensionEvalFunction_RunModuleExtensionResult( - accumulatedFileDigests, generatedRepoSpecs); + accumulatedFileDigests, generatedRepoSpecs, moduleExtensionMetadata); } } } - - diff --git a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py index ce5700804d38f9..8f55d869bafd19 100644 --- a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py +++ b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py @@ -736,5 +736,130 @@ def testModuleExtensionWithFile(self): self.assertIn('I have changed now!', ''.join(stderr)) + def testExtensionEvaluationDoesNotRerunOnChangedImports(self): + """ + + """ + self.ScratchFile( + 'MODULE.bazel', + [ + 'lockfile_ext = use_extension("extension.bzl", "lockfile_ext")', + 'use_repo(lockfile_ext, "dep", "indirect_dep", "invalid_dep")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\'lala\')")', + '', + 'repo_rule = repository_rule(implementation=_repo_rule_impl)', + '', + 'def _module_ext_impl(ctx):', + ' print("I am being evaluated")', + ' repo_rule(name="dep")', + ' repo_rule(name="missing_dep")', + ' repo_rule(name="indirect_dep")', + ' return ctx.extension_metadata(', + ' root_module_direct_deps=["dep", "missing_dep"],', + ' root_module_direct_dev_deps=[],', + ' )', + '', + 'lockfile_ext = module_extension(', + ' implementation=_module_ext_impl', + ')', + ], + ) + + # The build fails due to the "invalid_dep" import, which is not + # generated by the extension. + # Warnings should still be shown. + _, _, stderr = self.RunBazel(['build', '@dep//:all'], allow_failure=True) + stderr = '\n'.join(stderr) + self.assertIn('I am being evaluated', stderr) + self.assertIn( + 'Imported, but not created by the extension (will cause the build to' + ' fail):\ninvalid_dep', + stderr, + ) + self.assertIn( + 'Not imported, but reported as direct dependencies by the extension' + ' (may cause the build to fail):\nmissing_dep', + stderr, + ) + self.assertIn( + 'Imported, but reported as indirect dependencies by the' + ' extension:\nindirect_dep', + stderr, + ) + self.assertIn( + 'ERROR: module extension "lockfile_ext" from "//:extension.bzl" does' + ' not generate repository "invalid_dep"', + stderr, + ) + + # Shut down bazel to empty cache and run with no changes to verify + # that the warnings are still shown. + self.RunBazel(['shutdown']) + _, _, stderr = self.RunBazel(['build', '@dep//:all'], allow_failure=True) + stderr = '\n'.join(stderr) + self.assertNotIn('I am being evaluated', stderr) + self.assertIn( + 'Imported, but not created by the extension (will cause the build to' + ' fail):\ninvalid_dep', + stderr, + ) + self.assertIn( + 'Not imported, but reported as direct dependencies by the extension' + ' (may cause the build to fail):\nmissing_dep', + stderr, + ) + self.assertIn( + 'Imported, but reported as indirect dependencies by the' + ' extension:\nindirect_dep', + stderr, + ) + self.assertIn( + 'ERROR: module extension "lockfile_ext" from "//:extension.bzl" does' + ' not generate repository "invalid_dep"', + stderr, + ) + + # Fix the imports, which should not trigger a rerun of the extension + # even though imports and locations changed. + self.ScratchFile( + 'MODULE.bazel', + [ + '# This is a comment that changes the location of the usage below.', + 'lockfile_ext = use_extension("extension.bzl", "lockfile_ext")', + 'use_repo(lockfile_ext, "dep", "missing_dep")', + ], + ) + _, _, stderr = self.RunBazel(['build', '@dep//:all']) + stderr = '\n'.join(stderr) + self.assertNotIn('I am being evaluated', stderr) + self.assertNotIn( + 'Not imported, but reported as direct dependencies by the extension' + ' (may cause the build to fail):\nmissing_dep', + stderr, + ) + self.assertNotIn( + 'Imported, but reported as indirect dependencies by the' + ' extension:\nindirect_dep', + stderr, + ) + self.assertNotIn( + 'Imported, but reported as indirect dependencies by the' + ' extension:\nindirect_dep', + stderr, + ) + self.assertNotIn( + 'ERROR: module extension "lockfile_ext" from "//:extension.bzl" does' + ' not generate repository "invalid_dep"', + stderr, + ) + if __name__ == '__main__': unittest.main() From dea67d02c127e409d22546e20bf59c63ffe49946 Mon Sep 17 00:00:00 2001 From: "Ian (Hee) Cha" Date: Thu, 24 Aug 2023 09:26:16 -0700 Subject: [PATCH 034/125] [6.4.0] Add profiling for Bzlmod operations (#19313) The resolution of Bazel modules and evaluation of module extensions is now represented in the profile. Profile from a run of `bazel build //src:bazel-dev --enable_bzlmod --nobuild`: ![Bzlmod profile](https://github.com/bazelbuild/bazel/assets/4312191/22e03604-f832-4f64-be0e-a3281b94da3e) Closes #19291. Commit https://github.com/bazelbuild/bazel/commit/e7cfd2e9aeeaa4dfe7e1079eaa3b52421c4bb590#diff-602cd193eba2ecc3a616c8a5cb84e93d27ef4ee613d650139ff7bfa21bcee1f2 PiperOrigin-RevId: 559466489 Change-Id: If0989a4a6728f8dbf659197a7acbd3a0fcef9471 --------- Co-authored-by: Fabian Meumertzheim --- .../devtools/build/lib/bazel/bzlmod/BUILD | 2 + .../bazel/bzlmod/BazelLockFileFunction.java | 5 +- .../bzlmod/BazelModuleResolutionFunction.java | 61 +++++++++++++------ .../build/lib/bazel/bzlmod/IndexRegistry.java | 6 +- .../lib/bazel/bzlmod/ModuleExtensionId.java | 4 +- .../lib/bazel/bzlmod/ModuleFileFunction.java | 16 ++++- .../bzlmod/SingleExtensionEvalFunction.java | 9 ++- .../build/lib/profiler/ProfilerTask.java | 1 + .../lib/runtime/BlazeCommandDispatcher.java | 21 +++++-- 9 files changed, 95 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 60e7f592e856bd..b747623a3de343 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -80,6 +80,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", + "//src/main/java/com/google/devtools/build/lib/profiler", "//src/main/java/com/google/devtools/build/lib/util:os", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//third_party:caffeine", @@ -199,6 +200,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/packages", + "//src/main/java/com/google/devtools/build/lib/profiler", "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_directory_value", "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_function", "//src/main/java/com/google/devtools/build/lib/skyframe:bzl_load_value", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java index 1451f431cf916f..bbe02d643f6cf4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java @@ -22,6 +22,9 @@ import com.google.devtools.build.lib.actions.FileValue; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.cmdline.LabelConstants; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.profiler.SilentCloseable; import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed; import com.google.devtools.build.lib.vfs.FileSystemUtils; @@ -75,7 +78,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) return null; } - try { + try (SilentCloseable c = Profiler.instance().profile(ProfilerTask.BZLMOD, "parse lockfile")) { return getLockfileValue(lockfilePath); } catch (IOException | JsonSyntaxException | NullPointerException e) { throw new BazelLockfileFunctionException( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java index d84f09e6dd6eff..5f329ae1d45cde 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java @@ -29,6 +29,9 @@ import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.events.ExtendedEventHandler; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.profiler.SilentCloseable; import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed; import com.google.devtools.build.skyframe.SkyFunction; @@ -61,34 +64,52 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (root == null) { return null; } - ImmutableMap initialDepGraph = Discovery.run(env, root); - if (initialDepGraph == null) { - return null; + ImmutableMap initialDepGraph; + try (SilentCloseable c = Profiler.instance().profile(ProfilerTask.BZLMOD, "discovery")) { + initialDepGraph = Discovery.run(env, root); + if (initialDepGraph == null) { + return null; + } } Selection.Result selectionResult; - try { + try (SilentCloseable c = Profiler.instance().profile(ProfilerTask.BZLMOD, "selection")) { selectionResult = Selection.run(initialDepGraph, root.getOverrides()); } catch (ExternalDepsException e) { throw new BazelModuleResolutionFunctionException(e, Transience.PERSISTENT); } ImmutableMap resolvedDepGraph = selectionResult.getResolvedDepGraph(); - verifyRootModuleDirectDepsAreAccurate( - initialDepGraph.get(ModuleKey.ROOT), - resolvedDepGraph.get(ModuleKey.ROOT), - Objects.requireNonNull(CHECK_DIRECT_DEPENDENCIES.get(env)), - env.getListener()); + try (SilentCloseable c = + Profiler.instance().profile(ProfilerTask.BZLMOD, "verify root module direct deps")) { + verifyRootModuleDirectDepsAreAccurate( + initialDepGraph.get(ModuleKey.ROOT), + resolvedDepGraph.get(ModuleKey.ROOT), + Objects.requireNonNull(CHECK_DIRECT_DEPENDENCIES.get(env)), + env.getListener()); + } - checkBazelCompatibility( - resolvedDepGraph.values(), - Objects.requireNonNull(BAZEL_COMPATIBILITY_MODE.get(env)), - env.getListener()); + try (SilentCloseable c = + Profiler.instance().profile(ProfilerTask.BZLMOD, "check bazel compatibility")) { + checkBazelCompatibility( + resolvedDepGraph.values(), + Objects.requireNonNull(BAZEL_COMPATIBILITY_MODE.get(env)), + env.getListener()); + } - checkNoYankedVersions(resolvedDepGraph); + try (SilentCloseable c = + Profiler.instance().profile(ProfilerTask.BZLMOD, "check no yanked versions")) { + checkNoYankedVersions(resolvedDepGraph); + } + + ImmutableMap finalDepGraph; + try (SilentCloseable c = + Profiler.instance().profile(ProfilerTask.BZLMOD, "compute final dep graph")) { + finalDepGraph = + computeFinalDepGraph(resolvedDepGraph, root.getOverrides(), env.getListener()); + } - ImmutableMap finalDepGraph = - computeFinalDepGraph(resolvedDepGraph, root.getOverrides(), env.getListener()); + Profiler.instance().profile(ProfilerTask.BZLMOD, "module resolution completed").close(); return BazelModuleResolutionValue.create(finalDepGraph, selectionResult.getUnprunedDepGraph()); } @@ -241,6 +262,12 @@ private static RepoSpec computeRepoSpec( static Module moduleFromInterimModule( InterimModule interim, ModuleOverride override, ExtendedEventHandler eventHandler) throws BazelModuleResolutionFunctionException, InterruptedException { + RepoSpec repoSpec; + try (SilentCloseable c = + Profiler.instance() + .profile(ProfilerTask.BZLMOD, () -> "compute repo spec: " + interim.getKey())) { + repoSpec = computeRepoSpec(interim, override, eventHandler); + } return Module.builder() .setName(interim.getName()) .setVersion(interim.getVersion()) @@ -249,7 +276,7 @@ static Module moduleFromInterimModule( .setExecutionPlatformsToRegister(interim.getExecutionPlatformsToRegister()) .setToolchainsToRegister(interim.getToolchainsToRegister()) .setDeps(ImmutableMap.copyOf(Maps.transformValues(interim.getDeps(), DepSpec::toModuleKey))) - .setRepoSpec(computeRepoSpec(interim, override, eventHandler)) + .setRepoSpec(repoSpec) .setExtensionUsages(interim.getExtensionUsages()) .build(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java index 9e88a988f2602e..a3114b40137a5e 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java @@ -24,6 +24,9 @@ import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.ExtendedEventHandler; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.profiler.SilentCloseable; import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.gson.FieldNamingPolicy; @@ -80,7 +83,8 @@ private String constructUrl(String base, String... segments) { /** Grabs a file from the given URL. Returns {@link Optional#empty} if the file doesn't exist. */ private Optional grabFile(String url, ExtendedEventHandler eventHandler) throws IOException, InterruptedException { - try { + try (SilentCloseable c = + Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + url)) { return Optional.of( downloadManager.downloadAndReadOneUrl(new URL(url), eventHandler, clientEnv)); } catch (FileNotFoundException e) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionId.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionId.java index 250f83a27bcc65..cb02e4b3f2bf95 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionId.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionId.java @@ -76,7 +76,9 @@ public static ModuleExtensionId create( } public String asTargetString() { + String isolationKeyPart = getIsolationKey().map(key -> "%" + key).orElse(""); return String.format( - "%s%%%s", getBzlFileLabel().getUnambiguousCanonicalForm(), getExtensionName()); + "%s%%%s%s", + getBzlFileLabel().getUnambiguousCanonicalForm(), getExtensionName(), isolationKeyPart); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java index 2fec84fd7088cd..fbffb1b1d1d277 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java @@ -30,6 +30,9 @@ import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.packages.StarlarkExportable; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.profiler.SilentCloseable; import com.google.devtools.build.lib.rules.repository.RepositoryDirectoryValue; import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; @@ -130,8 +133,12 @@ public SkyValue compute(SkyKey skyKey, Environment env) ModuleFileValue.Key moduleFileKey = (ModuleFileValue.Key) skyKey; ModuleKey moduleKey = moduleFileKey.getModuleKey(); - GetModuleFileResult getModuleFileResult = - getModuleFile(moduleKey, moduleFileKey.getOverride(), allowedYankedVersions, env); + GetModuleFileResult getModuleFileResult; + try (SilentCloseable c = + Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "fetch module file: " + moduleKey)) { + getModuleFileResult = + getModuleFile(moduleKey, moduleFileKey.getOverride(), allowedYankedVersions, env); + } if (getModuleFileResult == null) { return null; } @@ -276,7 +283,10 @@ private ModuleFileGlobals execModuleFile( ModuleFileGlobals moduleFileGlobals = new ModuleFileGlobals(builtinModules, moduleKey, registry, ignoreDevDeps); - try (Mutability mu = Mutability.create("module file", moduleKey)) { + try (SilentCloseable c = + Profiler.instance() + .profile(ProfilerTask.BZLMOD, () -> "evaluate module file: " + moduleKey); + Mutability mu = Mutability.create("module file", moduleKey)) { net.starlark.java.eval.Module predeclaredEnv = getPredeclaredEnv(moduleFileGlobals, starlarkSemantics); Program program = Program.compileFile(starlarkFile, predeclaredEnv); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java index 8dbdadac7333ff..f9e4b641a55b70 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java @@ -35,6 +35,9 @@ import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.events.Event; +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.profiler.SilentCloseable; import com.google.devtools.build.lib.rules.repository.NeedsSkyframeRestartException; import com.google.devtools.build.lib.rules.repository.RepositoryFunction; import com.google.devtools.build.lib.runtime.ProcessWrapper; @@ -487,7 +490,11 @@ private RunModuleExtensionResult runModuleExtension( thread.setPrintHandler(Event.makeDebugPrintHandler(env.getListener())); moduleContext = createContext(env, usagesValue, starlarkSemantics, extensionId, extension); threadContext.storeInThread(thread); - try { + try (SilentCloseable c = + Profiler.instance() + .profile( + ProfilerTask.BZLMOD, + () -> "evaluate module extension: " + extensionId.asTargetString())) { Object returnValue = Starlark.fastcall( thread, extension.getImplementation(), new Object[] {moduleContext}, new Object[0]); diff --git a/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java b/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java index 4648f458f87825..7c47e6e220889c 100644 --- a/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java +++ b/src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java @@ -28,6 +28,7 @@ public enum ProfilerTask { ACTION_RELEASE("action resource release", Threshold.TEN_MILLIS), ACTION_UPDATE("update action information", Threshold.TEN_MILLIS), ACTION_COMPLETE("complete action execution"), + BZLMOD("bazel module processing"), INFO("general information"), CREATE_PACKAGE("package creation"), REMOTE_EXECUTION("remote action execution"), diff --git a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java index 47882616e1a85d..e2f83eefa6bc9e 100644 --- a/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java +++ b/src/main/java/com/google/devtools/build/lib/runtime/BlazeCommandDispatcher.java @@ -46,6 +46,7 @@ import com.google.devtools.build.lib.exec.ExecutionOptions; import com.google.devtools.build.lib.profiler.MemoryProfiler; import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.profiler.SilentCloseable; import com.google.devtools.build.lib.runtime.proto.InvocationPolicyOuterClass.InvocationPolicy; import com.google.devtools.build.lib.server.FailureDetails; @@ -584,7 +585,8 @@ private BlazeCommandResult execExclusively( // Compute the repo mapping of the main repo and re-parse options so that we get correct // values for label-typed options. env.getEventBus().post(new MainRepoMappingComputationStartingEvent()); - try { + try (SilentCloseable c = + Profiler.instance().profile(ProfilerTask.BZLMOD, "compute main repo mapping")) { RepositoryMapping mainRepoMapping = env.getSkyframeExecutor().getMainRepoMapping(reporter); optionsParser = optionsParser.toBuilder().withConversionContext(mainRepoMapping).build(); @@ -605,10 +607,14 @@ private BlazeCommandResult execExclusively( result = BlazeCommandResult.detailedExitCode(earlyExitCode); return result; } - optionHandler = - new BlazeOptionHandler( - runtime, workspace, command, commandAnnotation, optionsParser, invocationPolicy); - earlyExitCode = optionHandler.parseOptions(args, reporter); + try (SilentCloseable c = + Profiler.instance() + .profile(ProfilerTask.BZLMOD, "reparse options with main repo mapping")) { + optionHandler = + new BlazeOptionHandler( + runtime, workspace, command, commandAnnotation, optionsParser, invocationPolicy); + earlyExitCode = optionHandler.parseOptions(args, reporter); + } if (!earlyExitCode.isSuccess()) { reporter.post( new NoBuildEvent( @@ -619,7 +625,10 @@ private BlazeCommandResult execExclusively( } // Parse starlark options. - earlyExitCode = optionHandler.parseStarlarkOptions(env, reporter); + try (SilentCloseable c = + Profiler.instance().profile(ProfilerTask.BZLMOD, "parse starlark options")) { + earlyExitCode = optionHandler.parseStarlarkOptions(env, reporter); + } if (!earlyExitCode.isSuccess()) { reporter.post( new NoBuildEvent( From f5b0fd5dee70f5f2d2a5c85c4358d7d1924a6390 Mon Sep 17 00:00:00 2001 From: Chi Wang Date: Tue, 29 Aug 2023 10:46:30 +0200 Subject: [PATCH 035/125] [6.4.0] Retry on javax.net.ssl.SSLException ... BAD_DECRYPT (#19346) Cherry-picks two commits: - Retry on HTTP remote cache fetch failure (6115d94cd05864fe5c6e5f774e9482b3b4976976). - Retry on javax.net.ssl.SSLException ... BAD_DECRYPT (fb38c3aa8d3c7691e23f9e129d8986a06323a613). Fixes #19326. --------- Co-authored-by: Andreas Herrmann --- .../lib/remote/RemoteCacheClientFactory.java | 23 +- .../build/lib/remote/RemoteModule.java | 39 ++- .../devtools/build/lib/remote/http/BUILD | 1 + .../lib/remote/http/DownloadCommand.java | 12 +- .../lib/remote/http/HttpCacheClient.java | 65 +++-- .../lib/remote/http/HttpDownloadHandler.java | 16 ++ .../build/lib/remote/http/HttpException.java | 2 +- .../remote/RemoteCacheClientFactoryTest.java | 52 +++- .../devtools/build/lib/remote/http/BUILD | 1 + .../lib/remote/http/HttpCacheClientTest.java | 233 ++++++++++++++++-- .../remote/http/HttpDownloadHandlerTest.java | 52 ++++ 11 files changed, 451 insertions(+), 45 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactory.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactory.java index 46070133aecfa9..64f1bddf3b4c46 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactory.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactory.java @@ -59,15 +59,22 @@ public static RemoteCacheClient create( @Nullable Credentials creds, AuthAndTLSOptions authAndTlsOptions, Path workingDirectory, - DigestUtil digestUtil) + DigestUtil digestUtil, + RemoteRetrier retrier) throws IOException { Preconditions.checkNotNull(workingDirectory, "workingDirectory"); if (isHttpCache(options) && isDiskCache(options)) { return createDiskAndHttpCache( - workingDirectory, options.diskCache, options, creds, authAndTlsOptions, digestUtil); + workingDirectory, + options.diskCache, + options, + creds, + authAndTlsOptions, + digestUtil, + retrier); } if (isHttpCache(options)) { - return createHttp(options, creds, authAndTlsOptions, digestUtil); + return createHttp(options, creds, authAndTlsOptions, digestUtil, retrier); } if (isDiskCache(options)) { return createDiskCache( @@ -90,7 +97,8 @@ private static RemoteCacheClient createHttp( RemoteOptions options, Credentials creds, AuthAndTLSOptions authAndTlsOptions, - DigestUtil digestUtil) { + DigestUtil digestUtil, + RemoteRetrier retrier) { Preconditions.checkNotNull(options.remoteCache, "remoteCache"); try { @@ -109,6 +117,7 @@ private static RemoteCacheClient createHttp( options.remoteVerifyDownloads, ImmutableList.copyOf(options.remoteHeaders), digestUtil, + retrier, creds, authAndTlsOptions); } else { @@ -122,6 +131,7 @@ private static RemoteCacheClient createHttp( options.remoteVerifyDownloads, ImmutableList.copyOf(options.remoteHeaders), digestUtil, + retrier, creds, authAndTlsOptions); } @@ -148,9 +158,10 @@ private static RemoteCacheClient createDiskAndHttpCache( RemoteOptions options, Credentials cred, AuthAndTLSOptions authAndTlsOptions, - DigestUtil digestUtil) + DigestUtil digestUtil, + RemoteRetrier retrier) throws IOException { - RemoteCacheClient httpCache = createHttp(options, cred, authAndTlsOptions, digestUtil); + RemoteCacheClient httpCache = createHttp(options, cred, authAndTlsOptions, digestUtil, retrier); return createDiskAndRemoteClient( workingDirectory, diskCachePath, diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java index 18db228f3e254b..eb8a970384575c 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java @@ -66,6 +66,7 @@ import com.google.devtools.build.lib.remote.common.RemoteCacheClient; import com.google.devtools.build.lib.remote.common.RemoteExecutionClient; import com.google.devtools.build.lib.remote.downloader.GrpcRemoteDownloader; +import com.google.devtools.build.lib.remote.http.HttpException; import com.google.devtools.build.lib.remote.logging.LoggingInterceptor; import com.google.devtools.build.lib.remote.options.RemoteBuildEventUploadMode; import com.google.devtools.build.lib.remote.options.RemoteOptions; @@ -105,10 +106,13 @@ import io.grpc.Channel; import io.grpc.ClientInterceptor; import io.grpc.ManagedChannel; +import io.netty.handler.codec.DecoderException; +import io.netty.handler.codec.http.HttpResponseStatus; import io.reactivex.rxjava3.plugins.RxJavaPlugins; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.nio.channels.ClosedChannelException; import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutorService; @@ -116,6 +120,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; +import java.util.function.Predicate; import java.util.regex.Pattern; import javax.annotation.Nullable; @@ -221,6 +226,36 @@ private static ServerCapabilities getAndVerifyServerCapabilities( return capabilities; } + public static final Predicate RETRIABLE_HTTP_ERRORS = + e -> { + boolean retry = false; + if (e instanceof ClosedChannelException) { + retry = true; + } else if (e instanceof HttpException) { + int status = ((HttpException) e).response().status().code(); + retry = + status == HttpResponseStatus.INTERNAL_SERVER_ERROR.code() + || status == HttpResponseStatus.BAD_GATEWAY.code() + || status == HttpResponseStatus.SERVICE_UNAVAILABLE.code() + || status == HttpResponseStatus.GATEWAY_TIMEOUT.code(); + } else if (e instanceof IOException) { + String msg = e.getMessage().toLowerCase(); + if (msg.contains("connection reset by peer")) { + retry = true; + } else if (msg.contains("operation timed out")) { + retry = true; + } + } else { + // Workaround for a netty bug: https://github.com/netty/netty/issues/11815. Remove this + // once it is fixed in the upstream. + if (e instanceof DecoderException + && e.getMessage().endsWith("functions:OPENSSL_internal:BAD_DECRYPT")) { + retry = true; + } + } + return retry; + }; + private void initHttpAndDiskCache( CommandEnvironment env, Credentials credentials, @@ -235,7 +270,9 @@ private void initHttpAndDiskCache( credentials, authAndTlsOptions, Preconditions.checkNotNull(env.getWorkingDirectory(), "workingDirectory"), - digestUtil); + digestUtil, + new RemoteRetrier( + remoteOptions, RETRIABLE_HTTP_ERRORS, retryScheduler, Retrier.ALLOW_ALL_CALLS)); } catch (IOException e) { handleInitFailure(env, e, Code.CACHE_INIT_FAILURE); return; diff --git a/src/main/java/com/google/devtools/build/lib/remote/http/BUILD b/src/main/java/com/google/devtools/build/lib/remote/http/BUILD index 86b527cd6219f1..fb05bc0c1e5d37 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/http/BUILD +++ b/src/main/java/com/google/devtools/build/lib/remote/http/BUILD @@ -20,6 +20,7 @@ java_library( deps = [ "//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info", "//src/main/java/com/google/devtools/build/lib/authandtls", + "//src/main/java/com/google/devtools/build/lib/remote:Retrier", "//src/main/java/com/google/devtools/build/lib/remote/common", "//src/main/java/com/google/devtools/build/lib/remote/common:cache_not_found_exception", "//src/main/java/com/google/devtools/build/lib/remote/util", diff --git a/src/main/java/com/google/devtools/build/lib/remote/http/DownloadCommand.java b/src/main/java/com/google/devtools/build/lib/remote/http/DownloadCommand.java index a2e4abf9d83eb1..d68e7733f95cbc 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/http/DownloadCommand.java +++ b/src/main/java/com/google/devtools/build/lib/remote/http/DownloadCommand.java @@ -25,12 +25,18 @@ final class DownloadCommand { private final boolean casDownload; private final Digest digest; private final OutputStream out; + private final long offset; - DownloadCommand(URI uri, boolean casDownload, Digest digest, OutputStream out) { + DownloadCommand(URI uri, boolean casDownload, Digest digest, OutputStream out, long offset) { this.uri = Preconditions.checkNotNull(uri); this.casDownload = casDownload; this.digest = Preconditions.checkNotNull(digest); this.out = Preconditions.checkNotNull(out); + this.offset = offset; + } + + DownloadCommand(URI uri, boolean casDownload, Digest digest, OutputStream out) { + this(uri, casDownload, digest, out, 0); } public URI uri() { @@ -48,4 +54,8 @@ public Digest digest() { public OutputStream out() { return out; } + + public long offset() { + return offset; + } } diff --git a/src/main/java/com/google/devtools/build/lib/remote/http/HttpCacheClient.java b/src/main/java/com/google/devtools/build/lib/remote/http/HttpCacheClient.java index 6c6d4cc6f897fc..ff9b37ef16f542 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/http/HttpCacheClient.java +++ b/src/main/java/com/google/devtools/build/lib/remote/http/HttpCacheClient.java @@ -25,6 +25,7 @@ import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions; +import com.google.devtools.build.lib.remote.RemoteRetrier; import com.google.devtools.build.lib.remote.common.CacheNotFoundException; import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext; import com.google.devtools.build.lib.remote.common.RemoteCacheClient; @@ -81,8 +82,10 @@ import java.util.List; import java.util.Map.Entry; import java.util.NoSuchElementException; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import java.util.regex.Pattern; import javax.annotation.Nullable; @@ -129,6 +132,7 @@ public final class HttpCacheClient implements RemoteCacheClient { private final boolean useTls; private final boolean verifyDownloads; private final DigestUtil digestUtil; + private final RemoteRetrier retrier; private final Object closeLock = new Object(); @@ -150,6 +154,7 @@ public static HttpCacheClient create( boolean verifyDownloads, ImmutableList> extraHttpHeaders, DigestUtil digestUtil, + RemoteRetrier retrier, @Nullable final Credentials creds, AuthAndTLSOptions authAndTlsOptions) throws Exception { @@ -162,6 +167,7 @@ public static HttpCacheClient create( verifyDownloads, extraHttpHeaders, digestUtil, + retrier, creds, authAndTlsOptions, null); @@ -175,6 +181,7 @@ public static HttpCacheClient create( boolean verifyDownloads, ImmutableList> extraHttpHeaders, DigestUtil digestUtil, + RemoteRetrier retrier, @Nullable final Credentials creds, AuthAndTLSOptions authAndTlsOptions) throws Exception { @@ -189,6 +196,7 @@ public static HttpCacheClient create( verifyDownloads, extraHttpHeaders, digestUtil, + retrier, creds, authAndTlsOptions, domainSocketAddress); @@ -202,6 +210,7 @@ public static HttpCacheClient create( verifyDownloads, extraHttpHeaders, digestUtil, + retrier, creds, authAndTlsOptions, domainSocketAddress); @@ -219,6 +228,7 @@ private HttpCacheClient( boolean verifyDownloads, ImmutableList> extraHttpHeaders, DigestUtil digestUtil, + RemoteRetrier retrier, @Nullable final Credentials creds, AuthAndTLSOptions authAndTlsOptions, @Nullable SocketAddress socketAddress) @@ -284,6 +294,7 @@ public void channelCreated(Channel ch) { this.extraHttpHeaders = extraHttpHeaders; this.verifyDownloads = verifyDownloads; this.digestUtil = digestUtil; + this.retrier = retrier; } @SuppressWarnings("FutureReturnValueIgnored") @@ -441,8 +452,11 @@ public ListenableFuture downloadBlob( RemoteActionExecutionContext context, Digest digest, OutputStream out) { final DigestOutputStream digestOut = verifyDownloads ? digestUtil.newDigestOutputStream(out) : null; + final AtomicLong casBytesDownloaded = new AtomicLong(); return Futures.transformAsync( - get(digest, digestOut != null ? digestOut : out, /* casDownload= */ true), + retrier.executeAsync( + () -> + get(digest, digestOut != null ? digestOut : out, Optional.of(casBytesDownloaded))), (v) -> { try { if (digestOut != null) { @@ -458,7 +472,8 @@ public ListenableFuture downloadBlob( } @SuppressWarnings("FutureReturnValueIgnored") - private ListenableFuture get(Digest digest, final OutputStream out, boolean casDownload) { + private ListenableFuture get( + Digest digest, final OutputStream out, Optional casBytesDownloaded) { final AtomicBoolean dataWritten = new AtomicBoolean(); OutputStream wrappedOut = new OutputStream() { @@ -469,12 +484,18 @@ private ListenableFuture get(Digest digest, final OutputStream out, boolea @Override public void write(byte[] b, int offset, int length) throws IOException { dataWritten.set(true); + if (casBytesDownloaded.isPresent()) { + casBytesDownloaded.get().addAndGet(length); + } out.write(b, offset, length); } @Override public void write(int b) throws IOException { dataWritten.set(true); + if (casBytesDownloaded.isPresent()) { + casBytesDownloaded.get().incrementAndGet(); + } out.write(b); } @@ -483,7 +504,12 @@ public void flush() throws IOException { out.flush(); } }; - DownloadCommand downloadCmd = new DownloadCommand(uri, casDownload, digest, wrappedOut); + long offset = 0; + if (casBytesDownloaded.isPresent()) { + offset = casBytesDownloaded.get().get(); + } + DownloadCommand downloadCmd = + new DownloadCommand(uri, casBytesDownloaded.isPresent(), digest, wrappedOut, offset); SettableFuture outerF = SettableFuture.create(); acquireDownloadChannel() .addListener( @@ -575,8 +601,11 @@ private void getAfterCredentialRefresh(DownloadCommand cmd, SettableFuture public ListenableFuture downloadActionResult( RemoteActionExecutionContext context, ActionKey actionKey, boolean inlineOutErr) { return Futures.transform( - Utils.downloadAsActionResult( - actionKey, (digest, out) -> get(digest, out, /* casDownload= */ false)), + retrier.executeAsync( + () -> + Utils.downloadAsActionResult( + actionKey, + (digest, out) -> get(digest, out, /* casBytesDownloaded= */ Optional.empty()))), CachedActionResult::remote, MoreExecutors.directExecutor()); } @@ -670,20 +699,28 @@ private void uploadAfterCredentialRefresh(UploadCommand upload, SettableFuture uploadFile( RemoteActionExecutionContext context, Digest digest, Path file) { - try { - return uploadAsync( - digest.getHash(), digest.getSizeBytes(), file.getInputStream(), /* casUpload= */ true); - } catch (IOException e) { - // Can be thrown from file.getInputStream. - return Futures.immediateFailedFuture(e); - } + return retrier.executeAsync( + () -> { + try { + return uploadAsync( + digest.getHash(), + digest.getSizeBytes(), + file.getInputStream(), + /* casUpload= */ true); + } catch (IOException e) { + // Can be thrown from file.getInputStream. + return Futures.immediateFailedFuture(e); + } + }); } @Override public ListenableFuture uploadBlob( RemoteActionExecutionContext context, Digest digest, ByteString data) { - return uploadAsync( - digest.getHash(), digest.getSizeBytes(), data.newInput(), /* casUpload= */ true); + return retrier.executeAsync( + () -> + uploadAsync( + digest.getHash(), digest.getSizeBytes(), data.newInput(), /* casUpload= */ true)); } @Override diff --git a/src/main/java/com/google/devtools/build/lib/remote/http/HttpDownloadHandler.java b/src/main/java/com/google/devtools/build/lib/remote/http/HttpDownloadHandler.java index 50d83d138a1d66..a6ecc0c14543ab 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/http/HttpDownloadHandler.java +++ b/src/main/java/com/google/devtools/build/lib/remote/http/HttpDownloadHandler.java @@ -51,6 +51,8 @@ final class HttpDownloadHandler extends AbstractHttpHandler { private long contentLength = -1; /** the path header in the http request */ private String path; + /** the bytes to skip in a full or chunked response */ + private long skipBytes; public HttpDownloadHandler( Credentials credentials, ImmutableList> extraHttpHeaders) { @@ -105,6 +107,19 @@ protected void channelRead0(ChannelHandlerContext ctx, HttpObject msg) throws Ex ByteBuf content = ((HttpContent) msg).content(); int readableBytes = content.readableBytes(); + if (skipBytes > 0) { + int skipNow; + if (skipBytes < readableBytes) { + // readableBytes is an int, meaning skipBytes < readableBytes <= INT_MAX. + // So, this conversion is safe. + skipNow = (int) skipBytes; + } else { + skipNow = readableBytes; + } + content.readerIndex(content.readerIndex() + skipNow); + skipBytes -= skipNow; + readableBytes = readableBytes - skipNow; + } content.readBytes(out, readableBytes); bytesReceived += readableBytes; if (msg instanceof LastHttpContent) { @@ -137,6 +152,7 @@ public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) DownloadCommand cmd = (DownloadCommand) msg; out = cmd.out(); path = constructPath(cmd.uri(), cmd.digest().getHash(), cmd.casDownload()); + skipBytes = cmd.offset(); HttpRequest request = buildRequest(path, constructHost(cmd.uri())); addCredentialHeaders(request, cmd.uri()); addExtraRemoteHeaders(request); diff --git a/src/main/java/com/google/devtools/build/lib/remote/http/HttpException.java b/src/main/java/com/google/devtools/build/lib/remote/http/HttpException.java index 89fde56046a8d6..6a2bfd5a50b246 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/http/HttpException.java +++ b/src/main/java/com/google/devtools/build/lib/remote/http/HttpException.java @@ -18,7 +18,7 @@ import java.io.IOException; /** An exception that propagates the http status. */ -final class HttpException extends IOException { +public final class HttpException extends IOException { private final HttpResponse response; HttpException(HttpResponse response, String message, Throwable cause) { diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactoryTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactoryTest.java index 4a44a5acc7e688..1d762664b5c6e5 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactoryTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteCacheClientFactoryTest.java @@ -17,6 +17,8 @@ import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions; import com.google.devtools.build.lib.clock.JavaClock; import com.google.devtools.build.lib.remote.common.RemoteCacheClient; @@ -32,6 +34,7 @@ import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem; import com.google.devtools.common.options.Options; import java.io.IOException; +import java.util.concurrent.Executors; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -47,6 +50,14 @@ public class RemoteCacheClientFactoryTest { private final AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class); private Path workingDirectory; private InMemoryFileSystem fs; + private ListeningScheduledExecutorService retryScheduler = + MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1)); + private RemoteRetrier retrier = + new RemoteRetrier( + () -> RemoteRetrier.RETRIES_DISABLED, + (e) -> false, + retryScheduler, + Retrier.ALLOW_ALL_CALLS); @Before public final void setUp() { @@ -63,7 +74,12 @@ public void createCombinedCacheWithExistingWorkingDirectory() throws IOException RemoteCacheClient blobStore = RemoteCacheClientFactory.create( - remoteOptions, /* creds= */ null, authAndTlsOptions, workingDirectory, digestUtil); + remoteOptions, + /* creds= */ null, + authAndTlsOptions, + workingDirectory, + digestUtil, + retrier); assertThat(blobStore).isInstanceOf(DiskAndRemoteCacheClient.class); } @@ -76,7 +92,12 @@ public void createCombinedCacheWithNotExistingWorkingDirectory() throws IOExcept RemoteCacheClient blobStore = RemoteCacheClientFactory.create( - remoteOptions, /* creds= */ null, authAndTlsOptions, workingDirectory, digestUtil); + remoteOptions, + /* creds= */ null, + authAndTlsOptions, + workingDirectory, + digestUtil, + retrier); assertThat(blobStore).isInstanceOf(DiskAndRemoteCacheClient.class); assertThat(workingDirectory.exists()).isTrue(); @@ -96,7 +117,8 @@ public void createCombinedCacheWithMissingWorkingDirectoryShouldThrowException() /* creds= */ null, authAndTlsOptions, /* workingDirectory= */ null, - digestUtil)); + digestUtil, + retrier)); } @Test @@ -106,7 +128,12 @@ public void createHttpCacheWithProxy() throws IOException { RemoteCacheClient blobStore = RemoteCacheClientFactory.create( - remoteOptions, /* creds= */ null, authAndTlsOptions, workingDirectory, digestUtil); + remoteOptions, + /* creds= */ null, + authAndTlsOptions, + workingDirectory, + digestUtil, + retrier); assertThat(blobStore).isInstanceOf(HttpCacheClient.class); } @@ -125,7 +152,8 @@ public void createHttpCacheFailsWithUnsupportedProxyProtocol() { /* creds= */ null, authAndTlsOptions, workingDirectory, - digestUtil))) + digestUtil, + retrier))) .hasMessageThat() .contains("Remote cache proxy unsupported: bad-proxy"); } @@ -136,7 +164,12 @@ public void createHttpCacheWithoutProxy() throws IOException { RemoteCacheClient blobStore = RemoteCacheClientFactory.create( - remoteOptions, /* creds= */ null, authAndTlsOptions, workingDirectory, digestUtil); + remoteOptions, + /* creds= */ null, + authAndTlsOptions, + workingDirectory, + digestUtil, + retrier); assertThat(blobStore).isInstanceOf(HttpCacheClient.class); } @@ -147,7 +180,12 @@ public void createDiskCache() throws IOException { RemoteCacheClient blobStore = RemoteCacheClientFactory.create( - remoteOptions, /* creds= */ null, authAndTlsOptions, workingDirectory, digestUtil); + remoteOptions, + /* creds= */ null, + authAndTlsOptions, + workingDirectory, + digestUtil, + retrier); assertThat(blobStore).isInstanceOf(DiskCacheClient.class); } diff --git a/src/test/java/com/google/devtools/build/lib/remote/http/BUILD b/src/test/java/com/google/devtools/build/lib/remote/http/BUILD index 1486dda102a960..d0274727b8fca9 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/http/BUILD +++ b/src/test/java/com/google/devtools/build/lib/remote/http/BUILD @@ -21,6 +21,7 @@ java_test( test_class = "com.google.devtools.build.lib.AllTests", deps = [ "//src/main/java/com/google/devtools/build/lib/authandtls", + "//src/main/java/com/google/devtools/build/lib/remote:Retrier", "//src/main/java/com/google/devtools/build/lib/remote/common", "//src/main/java/com/google/devtools/build/lib/remote/http", "//src/main/java/com/google/devtools/build/lib/remote/util", diff --git a/src/test/java/com/google/devtools/build/lib/remote/http/HttpCacheClientTest.java b/src/test/java/com/google/devtools/build/lib/remote/http/HttpCacheClientTest.java index 41cddce1827849..cfac498e07b677 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/http/HttpCacheClientTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/http/HttpCacheClientTest.java @@ -27,13 +27,18 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import build.bazel.remote.execution.v2.ActionResult; import build.bazel.remote.execution.v2.Digest; import com.google.auth.Credentials; -import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ListeningScheduledExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import com.google.devtools.build.lib.authandtls.AuthAndTLSOptions; +import com.google.devtools.build.lib.remote.RemoteRetrier; +import com.google.devtools.build.lib.remote.Retrier; import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext; +import com.google.devtools.build.lib.remote.common.RemoteCacheClient; import com.google.devtools.build.lib.remote.util.DigestUtil; import com.google.devtools.build.lib.remote.util.TracingMetadataUtils; import com.google.devtools.build.lib.vfs.DigestHashFunction; @@ -44,6 +49,7 @@ import io.netty.bootstrap.ServerBootstrap; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufUtil; +import io.netty.buffer.Unpooled; import io.netty.channel.Channel; import io.netty.channel.ChannelFutureListener; import io.netty.channel.ChannelHandler; @@ -66,9 +72,13 @@ import io.netty.channel.unix.DomainSocketAddress; import io.netty.handler.codec.TooLongFrameException; import io.netty.handler.codec.http.DefaultFullHttpResponse; +import io.netty.handler.codec.http.DefaultHttpContent; +import io.netty.handler.codec.http.DefaultHttpResponse; +import io.netty.handler.codec.http.DefaultLastHttpContent; import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.codec.http.FullHttpResponse; import io.netty.handler.codec.http.HttpHeaderNames; +import io.netty.handler.codec.http.HttpHeaderValues; import io.netty.handler.codec.http.HttpObjectAggregator; import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpServerCodec; @@ -82,14 +92,16 @@ import java.net.InetSocketAddress; import java.net.SocketAddress; import java.net.URI; +import java.nio.channels.ClosedChannelException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; import java.util.function.IntFunction; import javax.annotation.Nullable; import org.junit.Before; @@ -155,6 +167,7 @@ interface TestServer { private static final class InetTestServer implements TestServer { + @Override public ServerChannel start(ChannelInboundHandler handler) { return createServer( NioServerSocketChannel.class, @@ -163,6 +176,7 @@ public ServerChannel start(ChannelInboundHandler handler) { handler); } + @Override public void stop(ServerChannel serverChannel) { try { serverChannel.close(); @@ -207,12 +221,14 @@ protected void initChannel(Channel ch) { } } + @Override public ServerChannel start(ChannelInboundHandler handler) { reset(this.serverChannel); this.handler = handler; return this.serverChannel; } + @Override public void stop(ServerChannel serverChannel) { // Note: In the tests, we expect that connecting to a closed server channel results // in a channel connection error. Netty doesn't seem to handle closing domain socket @@ -230,7 +246,7 @@ public void stop(ServerChannel serverChannel) { } @Parameters - public static Collection createInputValues() { + public static List createInputValues() { ArrayList parameters = new ArrayList(Arrays.asList(new Object[][] {{new InetTestServer()}})); @@ -262,9 +278,21 @@ private HttpCacheClient createHttpBlobStore( int timeoutSeconds, boolean remoteVerifyDownloads, @Nullable final Credentials creds, - AuthAndTLSOptions authAndTlsOptions) + AuthAndTLSOptions authAndTlsOptions, + Optional optRetrier) throws Exception { SocketAddress socketAddress = serverChannel.localAddress(); + RemoteRetrier retrier = + optRetrier.orElseGet( + () -> { + ListeningScheduledExecutorService retryScheduler = + MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1)); + return new RemoteRetrier( + () -> RemoteRetrier.RETRIES_DISABLED, + (e) -> false, + retryScheduler, + Retrier.ALLOW_ALL_CALLS); + }); if (socketAddress instanceof DomainSocketAddress) { DomainSocketAddress domainSocketAddress = (DomainSocketAddress) socketAddress; URI uri = new URI("http://localhost"); @@ -276,6 +304,7 @@ private HttpCacheClient createHttpBlobStore( remoteVerifyDownloads, ImmutableList.of(), DIGEST_UTIL, + retrier, creds, authAndTlsOptions); } else if (socketAddress instanceof InetSocketAddress) { @@ -288,6 +317,7 @@ private HttpCacheClient createHttpBlobStore( remoteVerifyDownloads, ImmutableList.of(), DIGEST_UTIL, + retrier, creds, authAndTlsOptions); } else { @@ -303,7 +333,12 @@ private HttpCacheClient createHttpBlobStore( AuthAndTLSOptions authAndTlsOptions) throws Exception { return createHttpBlobStore( - serverChannel, timeoutSeconds, /* remoteVerifyDownloads= */ true, creds, authAndTlsOptions); + serverChannel, + timeoutSeconds, + /* remoteVerifyDownloads= */ true, + creds, + authAndTlsOptions, + Optional.empty()); } @Before @@ -373,7 +408,7 @@ protected void channelRead0( AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class); HttpCacheClient blobStore = createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials, authAndTlsOptions); - byte[] data = "File Contents".getBytes(Charsets.US_ASCII); + byte[] data = "File Contents".getBytes(StandardCharsets.US_ASCII); assertThrows( UploadTimeoutException.class, () -> @@ -443,7 +478,7 @@ protected void channelRead0( AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class); HttpCacheClient blobStore = createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials, authAndTlsOptions); - ByteString data = ByteString.copyFrom("File Contents", Charsets.US_ASCII); + ByteString data = ByteString.copyFrom("File Contents", StandardCharsets.US_ASCII); IOException e = assertThrows( IOException.class, @@ -489,8 +524,9 @@ protected void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) /* timeoutSeconds= */ 1, /* remoteVerifyDownloads= */ true, credentials, - authAndTlsOptions); - Digest fooDigest = DIGEST_UTIL.compute("foo".getBytes(Charsets.UTF_8)); + authAndTlsOptions, + Optional.empty()); + Digest fooDigest = DIGEST_UTIL.compute("foo".getBytes(StandardCharsets.UTF_8)); try (OutputStream out = new ByteArrayOutputStream()) { IOException e = assertThrows( @@ -536,17 +572,144 @@ protected void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) /* timeoutSeconds= */ 1, /* remoteVerifyDownloads= */ false, credentials, - authAndTlsOptions); - Digest fooDigest = DIGEST_UTIL.compute("foo".getBytes(Charsets.UTF_8)); + authAndTlsOptions, + Optional.empty()); + Digest fooDigest = DIGEST_UTIL.compute("foo".getBytes(StandardCharsets.UTF_8)); try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { getFromFuture(blobStore.downloadBlob(remoteActionExecutionContext, fooDigest, out)); - assertThat(out.toByteArray()).isEqualTo("bar".getBytes(Charsets.UTF_8)); + assertThat(out.toByteArray()).isEqualTo("bar".getBytes(StandardCharsets.UTF_8)); } } finally { testServer.stop(server); } } + @Test + public void partialDownloadFailsWithoutRetry() throws Exception { + ServerChannel server = null; + try { + ByteBuf chunk1 = Unpooled.wrappedBuffer("File ".getBytes(StandardCharsets.US_ASCII)); + ByteBuf chunk2 = Unpooled.wrappedBuffer("Contents".getBytes(StandardCharsets.US_ASCII)); + server = testServer.start(new IntermittentFailureHandler(chunk1, chunk2)); + Credentials credentials = newCredentials(); + AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class); + + HttpCacheClient blobStore = + createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials, authAndTlsOptions); + assertThrows( + ClosedChannelException.class, + () -> + getFromFuture( + blobStore.downloadBlob( + remoteActionExecutionContext, DIGEST, new ByteArrayOutputStream()))); + } finally { + testServer.stop(server); + } + } + + @Test + public void partialDownloadSucceedsWithRetry() throws Exception { + ServerChannel server = null; + try { + ByteBuf chunk1 = Unpooled.wrappedBuffer("File ".getBytes(StandardCharsets.US_ASCII)); + // Replace first chunk to test that the client skips the redundant prefix on retry. + ByteBuf chunk1Attempt2 = Unpooled.wrappedBuffer("abcde".getBytes(StandardCharsets.US_ASCII)); + ByteBuf chunk2 = Unpooled.wrappedBuffer("Contents".getBytes(StandardCharsets.US_ASCII)); + server = testServer.start(new IntermittentFailureHandler(chunk1, chunk1Attempt2, chunk2)); + Credentials credentials = newCredentials(); + AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class); + + ListeningScheduledExecutorService retryScheduler = + MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1)); + RemoteRetrier retrier = + new RemoteRetrier( + () -> new Retrier.ZeroBackoff(1), + (e) -> { + return e instanceof ClosedChannelException; + }, + retryScheduler, + Retrier.ALLOW_ALL_CALLS); + HttpCacheClient blobStore = + createHttpBlobStore( + server, + /* timeoutSeconds= */ 1, + /* remoteVerifyDownloads= */ false, + credentials, + authAndTlsOptions, + Optional.of(retrier)); + + ByteArrayOutputStream download = new ByteArrayOutputStream(); + getFromFuture(blobStore.downloadBlob(remoteActionExecutionContext, DIGEST, download)); + assertThat(download.toByteArray()) + .isEqualTo("File Contents".getBytes(StandardCharsets.US_ASCII)); + } finally { + testServer.stop(server); + } + } + + @Test + public void actionResultRetryReadsFromStart() throws Exception { + ServerChannel server = null; + try { + ActionResult.Builder builder1 = ActionResult.newBuilder(); + builder1 + .addOutputFilesBuilder() + .setPath("attempt1/filename") + .setDigest(DIGEST_UTIL.computeAsUtf8("digest1")) + .setIsExecutable(true); + ActionResult action1 = builder1.build(); + ByteArrayOutputStream buffer1 = new ByteArrayOutputStream(); + action1.writeTo(buffer1); + int splitAt = buffer1.size() / 2; + ByteBuf chunk1 = Unpooled.copiedBuffer(buffer1.toByteArray(), 0, splitAt); + + // Replace first chunk to test that the client starts a fresh ActionResult download on retry. + ActionResult.Builder builder2 = ActionResult.newBuilder(); + builder2 + .addOutputFilesBuilder() + .setPath("attempt2/filename") + .setDigest(DIGEST_UTIL.computeAsUtf8("digest2")) + .setIsExecutable(false); + ActionResult action2 = builder2.build(); + ByteArrayOutputStream buffer2 = new ByteArrayOutputStream(); + action2.writeTo(buffer2); + ByteBuf chunk1Attempt2 = Unpooled.copiedBuffer(buffer2.toByteArray(), 0, splitAt); + ByteBuf chunk2 = + Unpooled.copiedBuffer(buffer2.toByteArray(), splitAt, buffer2.size() - splitAt); + + server = testServer.start(new IntermittentFailureHandler(chunk1, chunk1Attempt2, chunk2)); + Credentials credentials = newCredentials(); + AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class); + + ListeningScheduledExecutorService retryScheduler = + MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(1)); + RemoteRetrier retrier = + new RemoteRetrier( + () -> new Retrier.ZeroBackoff(1), + (e) -> { + return e instanceof ClosedChannelException; + }, + retryScheduler, + Retrier.ALLOW_ALL_CALLS); + HttpCacheClient blobStore = + createHttpBlobStore( + server, + /* timeoutSeconds= */ 1, + /* remoteVerifyDownloads= */ false, + credentials, + authAndTlsOptions, + Optional.of(retrier)); + + RemoteCacheClient.CachedActionResult download = + getFromFuture( + blobStore.downloadActionResult( + remoteActionExecutionContext, new RemoteCacheClient.ActionKey(DIGEST), false)); + assertThat(download.actionResult()).isEqualTo(action2); + } finally { + testServer.stop(server); + } + } + @Test public void expiredAuthTokensShouldBeRetried_get() throws Exception { expiredAuthTokensShouldBeRetried_get( @@ -567,7 +730,7 @@ private void expiredAuthTokensShouldBeRetried_get( createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials, authAndTlsOptions); ByteArrayOutputStream out = Mockito.spy(new ByteArrayOutputStream()); getFromFuture(blobStore.downloadBlob(remoteActionExecutionContext, DIGEST, out)); - assertThat(out.toString(Charsets.US_ASCII.name())).isEqualTo("File Contents"); + assertThat(out.toString(StandardCharsets.US_ASCII.name())).isEqualTo("File Contents"); verify(credentials, times(1)).refresh(); verify(credentials, times(2)).getRequestMetadata(any(URI.class)); verify(credentials, times(2)).hasRequestMetadata(); @@ -597,7 +760,7 @@ private void expiredAuthTokensShouldBeRetried_put( AuthAndTLSOptions authAndTlsOptions = Options.getDefaults(AuthAndTLSOptions.class); HttpCacheClient blobStore = createHttpBlobStore(server, /* timeoutSeconds= */ 1, credentials, authAndTlsOptions); - byte[] data = "File Contents".getBytes(Charsets.US_ASCII); + byte[] data = "File Contents".getBytes(StandardCharsets.US_ASCII); blobStore .uploadBlob( remoteActionExecutionContext, DIGEST_UTIL.compute(data), ByteString.copyFrom(data)) @@ -753,7 +916,7 @@ protected void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) return; } ByteBuf content = ctx.alloc().buffer(); - content.writeCharSequence("File Contents", Charsets.US_ASCII); + content.writeCharSequence("File Contents", StandardCharsets.US_ASCII); FullHttpResponse response = new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK, content); HttpUtil.setKeepAlive(response, true); @@ -769,4 +932,44 @@ protected void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) } } } + + /** + * {@link ChannelHandler} that on the first request returns a partial response and then closes the + * stream, and on any further requests returns a full response. + */ + @Sharable + static class IntermittentFailureHandler extends SimpleChannelInboundHandler { + private final ByteBuf attempt1Chunk1; + private final ByteBuf attempt2Chunk1; + private final ByteBuf attempt2Chunk2; + private int messageCount; + + public IntermittentFailureHandler( + ByteBuf attempt1Chunk1, ByteBuf attempt2Chunk1, ByteBuf attempt2Chunk2) { + this.attempt1Chunk1 = attempt1Chunk1; + this.attempt2Chunk1 = attempt2Chunk1; + this.attempt2Chunk2 = attempt2Chunk2; + } + + public IntermittentFailureHandler(ByteBuf chunk1, ByteBuf chunk2) { + this(chunk1.copy(), chunk1, chunk2); + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, FullHttpRequest request) { + DefaultHttpResponse response = + new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK); + response.headers().set(HttpHeaderNames.TRANSFER_ENCODING, HttpHeaderValues.CHUNKED); + ctx.write(response); + if (messageCount == 0) { + ctx.writeAndFlush(new DefaultHttpContent(attempt1Chunk1)) + .addListener(ChannelFutureListener.CLOSE); + } else { + ctx.writeAndFlush(new DefaultHttpContent(attempt2Chunk1)); + ctx.writeAndFlush(new DefaultLastHttpContent(attempt2Chunk2)) + .addListener(ChannelFutureListener.CLOSE); + } + ++messageCount; + } + } } diff --git a/src/test/java/com/google/devtools/build/lib/remote/http/HttpDownloadHandlerTest.java b/src/test/java/com/google/devtools/build/lib/remote/http/HttpDownloadHandlerTest.java index 35eecfbd90bcd6..22bb088cd32682 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/http/HttpDownloadHandlerTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/http/HttpDownloadHandlerTest.java @@ -162,4 +162,56 @@ public void httpErrorsWithContentAreSupported() throws IOException { verify(out, never()).close(); assertThat(ch.isOpen()).isFalse(); } + + /** Test that the handler correctly supports downloads at an offset, e.g. on retry. */ + @Test + public void downloadAtOffsetShouldWork() throws IOException { + EmbeddedChannel ch = new EmbeddedChannel(new HttpDownloadHandler(null, ImmutableList.of())); + ByteArrayOutputStream out = Mockito.spy(new ByteArrayOutputStream()); + DownloadCommand cmd = new DownloadCommand(CACHE_URI, true, DIGEST, out, 2); + ChannelPromise writePromise = ch.newPromise(); + ch.writeOneOutbound(cmd, writePromise); + + HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK); + response.headers().set(HttpHeaders.CONTENT_LENGTH, 5); + response.headers().set(HttpHeaders.CONNECTION, HttpHeaderValues.KEEP_ALIVE); + ch.writeInbound(response); + ByteBuf content = Unpooled.buffer(); + content.writeBytes(new byte[] {1, 2, 3, 4, 5}); + ch.writeInbound(new DefaultLastHttpContent(content)); + + assertThat(writePromise.isDone()).isTrue(); + assertThat(out.toByteArray()).isEqualTo(new byte[] {3, 4, 5}); + verify(out, never()).close(); + assertThat(ch.isActive()).isTrue(); + } + + /** Test that the handler correctly supports chunked downloads at an offset, e.g. on retry. */ + @Test + public void chunkedDownloadAtOffsetShouldWork() throws IOException { + EmbeddedChannel ch = new EmbeddedChannel(new HttpDownloadHandler(null, ImmutableList.of())); + ByteArrayOutputStream out = Mockito.spy(new ByteArrayOutputStream()); + DownloadCommand cmd = new DownloadCommand(CACHE_URI, true, DIGEST, out, 3); + ChannelPromise writePromise = ch.newPromise(); + ch.writeOneOutbound(cmd, writePromise); + + HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK); + response.headers().set(HttpHeaders.TRANSFER_ENCODING, HttpHeaderValues.CHUNKED); + response.headers().set(HttpHeaders.CONNECTION, HttpHeaderValues.KEEP_ALIVE); + ch.writeInbound(response); + ByteBuf content1 = Unpooled.buffer(); + content1.writeBytes(new byte[] {1, 2}); + ch.writeInbound(new DefaultHttpContent(content1)); + ByteBuf content2 = Unpooled.buffer(); + content2.writeBytes(new byte[] {3, 4}); + ch.writeInbound(new DefaultHttpContent(content2)); + ByteBuf content3 = Unpooled.buffer(); + content3.writeBytes(new byte[] {5}); + ch.writeInbound(new DefaultLastHttpContent(content3)); + + assertThat(writePromise.isDone()).isTrue(); + assertThat(out.toByteArray()).isEqualTo(new byte[] {4, 5}); + verify(out, never()).close(); + assertThat(ch.isActive()).isTrue(); + } } From 83c7ae054bb24f1a2c37814c0dadc24e884e68e7 Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Tue, 29 Aug 2023 12:28:49 +0200 Subject: [PATCH 036/125] [6.4.0] Fetch `RepoSpecs` in parallel (#19354) Computing `RepoSpec`s for all selected Bazel modules is on the critical path for the computation of the main repository mapping and thus benefits from parallelized downloads. On my machine, this change has the following effect on `bazel build //src:bazel-dev --enable_bzlmod --nobuild`: ``` before compute main repo mapping: 8s 127ms after compute main repo mapping: 4s 226ms ``` Closes https://github.com/bazelbuild/bazel/pull/19294. Commit https://github.com/bazelbuild/bazel/commit/8a683109239314aa561a8422bb060f9d5b8c33bb PiperOrigin-RevId: 559819452 Change-Id: Ieef957fcfe402c909d2863ba4a4ca3540781a56d --- .../lib/bazel/BazelRepositoryModule.java | 4 +- .../devtools/build/lib/bazel/bzlmod/BUILD | 3 + .../bzlmod/BazelModuleResolutionFunction.java | 130 ++++++++++-------- .../build/lib/bazel/bzlmod/RepoSpec.java | 3 +- .../lib/bazel/bzlmod/RepoSpecFunction.java | 73 ++++++++++ .../build/lib/bazel/bzlmod/RepoSpecKey.java | 52 +++++++ .../build/lib/skyframe/SkyFunctions.java | 1 + .../build/lib/analysis/util/AnalysisMock.java | 3 + .../bzlmod/BazelDepGraphFunctionTest.java | 1 + .../bzlmod/BazelLockFileFunctionTest.java | 1 + .../BazelModuleResolutionFunctionTest.java | 1 + .../bzlmod/BzlmodRepoRuleFunctionTest.java | 1 + .../build/lib/bazel/bzlmod/DiscoveryTest.java | 1 + .../bzlmod/ModuleExtensionResolutionTest.java | 1 + .../bazel/bzlmod/ModuleFileFunctionTest.java | 1 + .../repository/RepositoryDelegatorTest.java | 2 + 16 files changed, 222 insertions(+), 56 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecFunction.java create mode 100644 src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecKey.java diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index d6621ccab8948e..3bd09647a4fbc4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -46,6 +46,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.RegistryFactory; import com.google.devtools.build.lib.bazel.bzlmod.RegistryFactoryImpl; import com.google.devtools.build.lib.bazel.bzlmod.RepoSpec; +import com.google.devtools.build.lib.bazel.bzlmod.RepoSpecFunction; import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionEvalFunction; import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionUsagesFunction; import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; @@ -272,7 +273,8 @@ SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace .addSkyFunction(SkyFunctions.BAZEL_MODULE_INSPECTION, new BazelModuleInspectorFunction()) .addSkyFunction(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .addSkyFunction(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction) - .addSkyFunction(SkyFunctions.SINGLE_EXTENSION_USAGES, new SingleExtensionUsagesFunction()); + .addSkyFunction(SkyFunctions.SINGLE_EXTENSION_USAGES, new SingleExtensionUsagesFunction()) + .addSkyFunction(SkyFunctions.REPO_SPEC, new RepoSpecFunction(registryFactory)); filesystem = runtime.getFileSystem(); credentialModule = Preconditions.checkNotNull(runtime.getBlazeModule(CredentialModule.class)); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index b747623a3de343..0830f09d87d558 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -24,6 +24,7 @@ java_library( ], deps = [ "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//src/main/java/net/starlark/java/eval", "//third_party:auto_value", "//third_party:gson", @@ -131,6 +132,7 @@ java_library( "MultipleVersionOverride.java", "NonRegistryOverride.java", "RegistryOverride.java", + "RepoSpecKey.java", "SingleExtensionEvalValue.java", "SingleExtensionUsagesValue.java", "SingleVersionOverride.java", @@ -174,6 +176,7 @@ java_library( "ModuleExtensionContext.java", "ModuleFileFunction.java", "ModuleFileGlobals.java", + "RepoSpecFunction.java", "Selection.java", "SingleExtensionEvalFunction.java", "SingleExtensionUsagesFunction.java", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java index 5f329ae1d45cde..dc9f79692bdf44 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java @@ -15,10 +15,13 @@ package com.google.devtools.build.lib.bazel.bzlmod; +import static com.google.common.collect.ImmutableSet.toImmutableSet; + import com.google.common.base.Strings; import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import com.google.devtools.build.lib.analysis.BlazeVersionInfo; import com.google.devtools.build.lib.bazel.BazelVersion; @@ -29,6 +32,7 @@ import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.EventHandler; import com.google.devtools.build.lib.events.ExtendedEventHandler; +import com.google.devtools.build.lib.events.StoredEventHandler; import com.google.devtools.build.lib.profiler.Profiler; import com.google.devtools.build.lib.profiler.ProfilerTask; import com.google.devtools.build.lib.profiler.SilentCloseable; @@ -39,7 +43,7 @@ import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; -import java.io.IOException; +import com.google.devtools.build.skyframe.SkyframeLookupResult; import java.util.Map; import java.util.Objects; import javax.annotation.Nullable; @@ -64,12 +68,61 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (root == null) { return null; } + + var state = env.getState(ModuleResolutionComputeState::new); + try { + if (state.selectionResult == null) { + state.storedEventHandler = new StoredEventHandler(); + state.selectionResult = discoverAndSelect(env, root, state.storedEventHandler); + if (state.selectionResult == null) { + return null; + } + } + } finally { + state.storedEventHandler.replayOn(env.getListener()); + } + + ImmutableSet repoSpecKeys = + state.selectionResult.getResolvedDepGraph().values().stream() + // Modules with a null registry have a non-registry override. We don't need to + // fetch or store the repo spec in this case. + .filter(module -> module.getRegistry() != null) + .map(RepoSpecKey::of) + .collect(toImmutableSet()); + SkyframeLookupResult repoSpecResults = env.getValuesAndExceptions(repoSpecKeys); + ImmutableMap.Builder remoteRepoSpecs = ImmutableMap.builder(); + for (RepoSpecKey repoSpecKey : repoSpecKeys) { + RepoSpec repoSpec = (RepoSpec) repoSpecResults.get(repoSpecKey); + if (repoSpec == null) { + return null; + } + remoteRepoSpecs.put(repoSpecKey.getModuleKey(), repoSpec); + } + + ImmutableMap finalDepGraph; + try (SilentCloseable c = + Profiler.instance().profile(ProfilerTask.BZLMOD, "compute final dep graph")) { + finalDepGraph = + computeFinalDepGraph( + state.selectionResult.getResolvedDepGraph(), + root.getOverrides(), + remoteRepoSpecs.buildOrThrow()); + } + + return BazelModuleResolutionValue.create( + finalDepGraph, state.selectionResult.getUnprunedDepGraph()); + } + + @Nullable + private static Selection.Result discoverAndSelect( + Environment env, RootModuleFileValue root, ExtendedEventHandler eventHandler) + throws BazelModuleResolutionFunctionException, InterruptedException { ImmutableMap initialDepGraph; try (SilentCloseable c = Profiler.instance().profile(ProfilerTask.BZLMOD, "discovery")) { initialDepGraph = Discovery.run(env, root); - if (initialDepGraph == null) { - return null; - } + } + if (initialDepGraph == null) { + return null; } Selection.Result selectionResult; @@ -86,7 +139,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) initialDepGraph.get(ModuleKey.ROOT), resolvedDepGraph.get(ModuleKey.ROOT), Objects.requireNonNull(CHECK_DIRECT_DEPENDENCIES.get(env)), - env.getListener()); + eventHandler); } try (SilentCloseable c = @@ -94,7 +147,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) checkBazelCompatibility( resolvedDepGraph.values(), Objects.requireNonNull(BAZEL_COMPATIBILITY_MODE.get(env)), - env.getListener()); + eventHandler); } try (SilentCloseable c = @@ -102,16 +155,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) checkNoYankedVersions(resolvedDepGraph); } - ImmutableMap finalDepGraph; - try (SilentCloseable c = - Profiler.instance().profile(ProfilerTask.BZLMOD, "compute final dep graph")) { - finalDepGraph = - computeFinalDepGraph(resolvedDepGraph, root.getOverrides(), env.getListener()); - } - - Profiler.instance().profile(ProfilerTask.BZLMOD, "module resolution completed").close(); - - return BazelModuleResolutionValue.create(finalDepGraph, selectionResult.getUnprunedDepGraph()); + return selectionResult; } private static void verifyRootModuleDirectDepsAreAccurate( @@ -210,7 +254,8 @@ private static void checkNoYankedVersions(ImmutableMap } } - private static RepoSpec maybeAppendAdditionalPatches(RepoSpec repoSpec, ModuleOverride override) { + private static RepoSpec maybeAppendAdditionalPatches( + @Nullable RepoSpec repoSpec, @Nullable ModuleOverride override) { if (!(override instanceof SingleVersionOverride)) { return repoSpec; } @@ -230,44 +275,15 @@ private static RepoSpec maybeAppendAdditionalPatches(RepoSpec repoSpec, ModuleOv .build(); } - @Nullable - private static RepoSpec computeRepoSpec( - InterimModule interimModule, ModuleOverride override, ExtendedEventHandler eventHandler) - throws BazelModuleResolutionFunctionException, InterruptedException { - if (interimModule.getRegistry() == null) { - // This module has a non-registry override. We don't need to store the repo spec in this case. - return null; - } - try { - RepoSpec moduleRepoSpec = - interimModule - .getRegistry() - .getRepoSpec( - interimModule.getKey(), interimModule.getCanonicalRepoName(), eventHandler); - return maybeAppendAdditionalPatches(moduleRepoSpec, override); - } catch (IOException e) { - throw new BazelModuleResolutionFunctionException( - ExternalDepsException.withMessage( - Code.ERROR_ACCESSING_REGISTRY, - "Unable to get module repo spec from registry: %s", - e.getMessage()), - Transience.PERSISTENT); - } - } - /** * Builds a {@link Module} from an {@link InterimModule}, discarding unnecessary fields and adding * extra necessary ones (such as the repo spec). + * + * @param remoteRepoSpec the {@link RepoSpec} for the module obtained from a registry or null if + * the module has a non-registry override */ static Module moduleFromInterimModule( - InterimModule interim, ModuleOverride override, ExtendedEventHandler eventHandler) - throws BazelModuleResolutionFunctionException, InterruptedException { - RepoSpec repoSpec; - try (SilentCloseable c = - Profiler.instance() - .profile(ProfilerTask.BZLMOD, () -> "compute repo spec: " + interim.getKey())) { - repoSpec = computeRepoSpec(interim, override, eventHandler); - } + InterimModule interim, @Nullable ModuleOverride override, @Nullable RepoSpec remoteRepoSpec) { return Module.builder() .setName(interim.getName()) .setVersion(interim.getVersion()) @@ -276,7 +292,7 @@ static Module moduleFromInterimModule( .setExecutionPlatformsToRegister(interim.getExecutionPlatformsToRegister()) .setToolchainsToRegister(interim.getToolchainsToRegister()) .setDeps(ImmutableMap.copyOf(Maps.transformValues(interim.getDeps(), DepSpec::toModuleKey))) - .setRepoSpec(repoSpec) + .setRepoSpec(maybeAppendAdditionalPatches(remoteRepoSpec, override)) .setExtensionUsages(interim.getExtensionUsages()) .build(); } @@ -284,18 +300,24 @@ static Module moduleFromInterimModule( private static ImmutableMap computeFinalDepGraph( ImmutableMap resolvedDepGraph, ImmutableMap overrides, - ExtendedEventHandler eventHandler) - throws BazelModuleResolutionFunctionException, InterruptedException { + ImmutableMap remoteRepoSpecs) { ImmutableMap.Builder finalDepGraph = ImmutableMap.builder(); for (Map.Entry entry : resolvedDepGraph.entrySet()) { finalDepGraph.put( entry.getKey(), moduleFromInterimModule( - entry.getValue(), overrides.get(entry.getKey().getName()), eventHandler)); + entry.getValue(), + overrides.get(entry.getKey().getName()), + remoteRepoSpecs.get(entry.getKey()))); } return finalDepGraph.buildOrThrow(); } + private static class ModuleResolutionComputeState implements Environment.SkyKeyComputeState { + Selection.Result selectionResult; + StoredEventHandler storedEventHandler; + } + static class BazelModuleResolutionFunctionException extends SkyFunctionException { BazelModuleResolutionFunctionException(ExternalDepsException e, Transience transience) { super(e, transience); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java index f5f2e6f851c8b7..41fcf284f52423 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpec.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.bazel.bzlmod; import com.google.auto.value.AutoValue; +import com.google.devtools.build.skyframe.SkyValue; import com.ryanharter.auto.value.gson.GenerateTypeAdapter; import javax.annotation.Nullable; @@ -24,7 +25,7 @@ */ @AutoValue @GenerateTypeAdapter -public abstract class RepoSpec { +public abstract class RepoSpec implements SkyValue { /** * The label string for the bzl file this repository rule is defined in, empty for native rule. diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecFunction.java new file mode 100644 index 00000000000000..0ac65cfb432008 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecFunction.java @@ -0,0 +1,73 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.devtools.build.lib.profiler.Profiler; +import com.google.devtools.build.lib.profiler.ProfilerTask; +import com.google.devtools.build.lib.profiler.SilentCloseable; +import com.google.devtools.build.lib.server.FailureDetails; +import com.google.devtools.build.skyframe.SkyFunction; +import com.google.devtools.build.skyframe.SkyFunctionException; +import com.google.devtools.build.skyframe.SkyKey; +import com.google.devtools.build.skyframe.SkyValue; +import java.io.IOException; +import java.net.URISyntaxException; +import javax.annotation.Nullable; + +/** + * A simple SkyFunction that computes a {@link RepoSpec} for the given {@link InterimModule} by + * fetching required information from its {@link Registry}. + */ +public class RepoSpecFunction implements SkyFunction { + private final RegistryFactory registryFactory; + + public RepoSpecFunction(RegistryFactory registryFactory) { + this.registryFactory = registryFactory; + } + + @Override + @Nullable + public SkyValue compute(SkyKey skyKey, Environment env) + throws InterruptedException, RepoSpecException { + RepoSpecKey key = (RepoSpecKey) skyKey.argument(); + try (SilentCloseable c = + Profiler.instance() + .profile(ProfilerTask.BZLMOD, () -> "compute repo spec: " + key.getModuleKey())) { + return registryFactory + .getRegistryWithUrl(key.getRegistryUrl()) + .getRepoSpec( + key.getModuleKey(), key.getModuleKey().getCanonicalRepoName(), env.getListener()); + } catch (IOException e) { + throw new RepoSpecException( + ExternalDepsException.withCauseAndMessage( + FailureDetails.ExternalDeps.Code.ERROR_ACCESSING_REGISTRY, + e, + "Unable to get module repo spec for %s from registry", + key.getModuleKey())); + } catch (URISyntaxException e) { + // This should never happen since we obtain the registry URL from an already constructed + // registry. + throw new IllegalStateException(e); + } + } + + static final class RepoSpecException extends SkyFunctionException { + + RepoSpecException(ExternalDepsException cause) { + super(cause, Transience.TRANSIENT); + } + } +} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecKey.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecKey.java new file mode 100644 index 00000000000000..fb39e3b3df9b30 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecKey.java @@ -0,0 +1,52 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package com.google.devtools.build.lib.bazel.bzlmod; + +import com.google.auto.value.AutoValue; +import com.google.common.base.Preconditions; +import com.google.common.collect.Interner; +import com.google.devtools.build.lib.concurrent.BlazeInterners; +import com.google.devtools.build.lib.skyframe.SkyFunctions; +import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec; +import com.google.devtools.build.skyframe.SkyFunctionName; +import com.google.devtools.build.skyframe.SkyKey; + +/** The key for {@link RepoSpecFunction}. */ +@AutoCodec +@AutoValue +abstract class RepoSpecKey implements SkyKey { + private static final Interner interner = BlazeInterners.newWeakInterner(); + + static RepoSpecKey of(InterimModule module) { + Preconditions.checkNotNull( + module.getRegistry(), "module must not have a non-registry override"); + return create(module.getKey(), module.getRegistry().getUrl()); + } + + abstract ModuleKey getModuleKey(); + + abstract String getRegistryUrl(); + + @AutoCodec.Instantiator + static RepoSpecKey create(ModuleKey moduleKey, String registryUrl) { + return interner.intern(new AutoValue_RepoSpecKey(moduleKey, registryUrl)); + } + + @Override + public SkyFunctionName functionName() { + return SkyFunctions.REPO_SPEC; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java index 9eb8b4f11577e3..29f68bfa2c510d 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyFunctions.java @@ -155,6 +155,7 @@ public final class SkyFunctions { SkyFunctionName.createHermetic("BAZEL_DEP_GRAPH"); public static final SkyFunctionName BAZEL_LOCK_FILE = SkyFunctionName.createHermetic("BAZEL_LOCK_FILE"); + public static final SkyFunctionName REPO_SPEC = SkyFunctionName.createNonHermetic("REPO_SPEC"); public static Predicate isSkyFunction(SkyFunctionName functionName) { return key -> key.functionName().equals(functionName); diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java index 047c2aa1741b70..28e2f54c77743f 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java @@ -23,6 +23,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; import com.google.devtools.build.lib.bazel.bzlmod.NonRegistryOverride; +import com.google.devtools.build.lib.bazel.bzlmod.RepoSpecFunction; import com.google.devtools.build.lib.bazel.rules.android.AndroidNdkRepositoryFunction; import com.google.devtools.build.lib.bazel.rules.android.AndroidNdkRepositoryRule; import com.google.devtools.build.lib.bazel.rules.android.AndroidSdkRepositoryFunction; @@ -148,6 +149,8 @@ public ImmutableMap getSkyFunctions(BlazeDirectori new BazelLockFileFunction(directories.getWorkspace()), SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction(), + SkyFunctions.REPO_SPEC, + new RepoSpecFunction(FakeRegistry.DEFAULT_FACTORY), SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, new ClientEnvironmentFunction(new AtomicReference<>(ImmutableMap.of())), CcSkyframeFdoSupportValue.SKYFUNCTION, diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java index b8602dba90d16e..8228dcac739c0b 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java @@ -129,6 +129,7 @@ public void setup() throws Exception { .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, resolutionFunctionMock) + .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction(registryFactory)) .put( SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, new ClientEnvironmentFunction( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java index 8956b9795dd4a5..2b7a308b4a5a20 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java @@ -156,6 +156,7 @@ public void setup() throws Exception { SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, rootDirectory, ImmutableMap.of())) .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) + .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction(registryFactory)) .put( SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, new ClientEnvironmentFunction( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java index 088943ef9a47e9..11041182aa9c00 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java @@ -115,6 +115,7 @@ public void setup() throws Exception { .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) + .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction(registryFactory)) .put( SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, new ClientEnvironmentFunction(new AtomicReference<>(ImmutableMap.of()))) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java index 46b9ae30be2aa2..adba54c010611c 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java @@ -130,6 +130,7 @@ public void setup() throws Exception { .put( SkyFunctions.MODULE_FILE, new ModuleFileFunction(registryFactory, workspaceRoot, ImmutableMap.of())) + .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction(registryFactory)) .put( SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, new ClientEnvironmentFunction(new AtomicReference<>(ImmutableMap.of()))) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java index 37ed209fa7e41b..d8d885cd5c465b 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java @@ -179,6 +179,7 @@ private void setUpWithBuiltinModules(ImmutableMap b .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) + .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction(registryFactory)) .put( SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, new ClientEnvironmentFunction(new AtomicReference<>(ImmutableMap.of()))) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java index 9d9faa84b4e904..009f78efa1c555 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java @@ -255,6 +255,7 @@ public void setup() throws Exception { .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) .put(SkyFunctions.SINGLE_EXTENSION_USAGES, new SingleExtensionUsagesFunction()) .put(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction) + .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction(registryFactory)) .put( SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, new ClientEnvironmentFunction(new AtomicReference<>(ImmutableMap.of()))) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java index 761f9142ed9022..b86b0b43a55822 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java @@ -153,6 +153,7 @@ private void setUpWithBuiltinModules(ImmutableMap b .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) + .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction(registryFactory)) .put( SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, new ClientEnvironmentFunction(new AtomicReference<>(ImmutableMap.of()))) diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java index d75121b3964689..873d79b915a4a0 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java @@ -36,6 +36,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue; import com.google.devtools.build.lib.bazel.bzlmod.FakeRegistry; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; +import com.google.devtools.build.lib.bazel.bzlmod.RepoSpecFunction; import com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; @@ -245,6 +246,7 @@ public void setupDelegator() throws Exception { .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) + .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction(registryFactory)) .put( SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, new ClientEnvironmentFunction(new AtomicReference<>(ImmutableMap.of()))) From cac82a01f2d13d2297258298f366a661976b2919 Mon Sep 17 00:00:00 2001 From: "Ian (Hee) Cha" Date: Thu, 31 Aug 2023 00:57:55 -0700 Subject: [PATCH 037/125] [6.4.0] Make `MODULE.bazel.lock` deterministic (#19370) Sorts the extensions, which would otherwise be added in the non-deterministic order in which module extension resolution events are fired, by their ID. Since Gson doesn't support custom comparators on a `TreeMap`, `ModuleExtensionId` now implements `Comparable` directly. Closes #19339. Commit https://github.com/bazelbuild/bazel/commit/ede4d49669dd305b10cee6f87d3b6146ebf10875 PiperOrigin-RevId: 560959804 Change-Id: Ic5579c032e6cd32fe4bd2114378812c0110109c2 Co-authored-by: Fabian Meumertzheim --- .../build/lib/bazel/bzlmod/BazelLockFileModule.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java index fd2bfddd42120a..a346f5981475ad 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java @@ -17,6 +17,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedMap; import com.google.common.eventbus.Subscribe; import com.google.common.flogger.GoogleLogger; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; @@ -123,7 +124,11 @@ private ImmutableMap combineModuleEx for (ModuleExtensionResolutionEvent extensionEvent : extensionResolutionEvents) { updatedExtensionMap.put(extensionEvent.getExtensionId(), extensionEvent.getModuleExtension()); } - return updatedExtensionMap.buildKeepingLast(); + // The order in which extensions are added to extensionResolutionEvents depends on the order + // in which their Skyframe evaluations finish, which is non-deterministic. We ensure a + // deterministic lockfile by sorting. + return ImmutableSortedMap.copyOf( + updatedExtensionMap.buildKeepingLast(), ModuleExtensionId.LEXICOGRAPHIC_COMPARATOR); } /** From 038d9738fe1466aa8096c382ee8137ea2fe40c01 Mon Sep 17 00:00:00 2001 From: "Ian (Hee) Cha" Date: Thu, 31 Aug 2023 01:49:12 -0700 Subject: [PATCH 038/125] [6.4.0] Ensure lockfile is updated after reset to pre-build state (#19371) The events that update the lockfile are important for incremental correctness and thus need to return `true` from `storeForReplay`. Before this change, if a build creates the lockfile because it didn't exist and the lockfile is then deleted, subsequent builds did not regenerate it as the relevant SkyFunctions wouldn't rerun and the events were not replayed. Closes #19343. Commit https://github.com/bazelbuild/bazel/commit/19c0c809abbe4bc70d3d6b493ff966dd41c54768 PiperOrigin-RevId: 561287438 Change-Id: I549f99b896a0095e8ffc35b7bacc8a841a44219a Co-authored-by: Salma Samy --- .../bzlmod/BazelModuleResolutionEvent.java | 5 +++ .../ModuleExtensionResolutionEvent.java | 5 +++ .../py/bazel/bzlmod/bazel_lockfile_test.py | 42 +++++++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java index 81a0dbaa144b9c..b2e0f6f3a5b40f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java @@ -36,4 +36,9 @@ public static BazelModuleResolutionEvent create( public abstract ImmutableTable getExtensionUsagesById(); + + @Override + public boolean storeForReplay() { + return true; + } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionEvent.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionEvent.java index 4013e7bb4c5965..cb82ae039f08fe 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionEvent.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionEvent.java @@ -33,4 +33,9 @@ public static ModuleExtensionResolutionEvent create( public abstract ModuleExtensionId getExtensionId(); public abstract LockFileModuleExtension getModuleExtension(); + + @Override + public boolean storeForReplay() { + return true; + } } diff --git a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py index 8f55d869bafd19..ed6212e4170623 100644 --- a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py +++ b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py @@ -861,5 +861,47 @@ def testExtensionEvaluationDoesNotRerunOnChangedImports(self): stderr, ) + def testLockfileRecreatedAfterDeletion(self): + self.ScratchFile( + 'MODULE.bazel', + [ + 'lockfile_ext = use_extension("extension.bzl", "lockfile_ext")', + 'use_repo(lockfile_ext, "hello")', + ], + ) + self.ScratchFile('BUILD.bazel') + self.ScratchFile( + 'extension.bzl', + [ + 'def _repo_rule_impl(ctx):', + ' ctx.file("WORKSPACE")', + ' ctx.file("BUILD", "filegroup(name=\'lala\')")', + '', + 'repo_rule = repository_rule(implementation=_repo_rule_impl)', + '', + 'def _module_ext_impl(ctx):', + ' repo_rule(name="hello")', + '', + 'lockfile_ext = module_extension(', + ' implementation=_module_ext_impl,', + ')', + ], + ) + + self.RunBazel(['build', '@hello//:all']) + + # Return the lockfile to the state it had before the + # previous build: it didn't exist. + with open('MODULE.bazel.lock', 'r') as lock_file: + old_data = lock_file.read() + os.remove('MODULE.bazel.lock') + + self.RunBazel(['build', '@hello//:all']) + + with open('MODULE.bazel.lock', 'r') as lock_file: + new_data = lock_file.read() + + self.assertEqual(old_data, new_data) + if __name__ == '__main__': unittest.main() From 1cd0ab05e1c8b4117a8ca70901716dd034176098 Mon Sep 17 00:00:00 2001 From: "bazel.build machine account" <15028808+bazel-io@users.noreply.github.com> Date: Thu, 31 Aug 2023 15:54:49 -0400 Subject: [PATCH 039/125] [6.4.0] build-runfiles: remove temporary file prior to creating it (#19386) When building with Remote Output Service, bb_clientd has the ability to restore snapshots of previous bazel-out/ directories. Though the file contents of these snapshots are identical to what's created in the past, the files will be read-only. This is because the files may be shared by multiple snapshots. We have noticed that most of Bazel is fine with that. Most of the times Bazel is a good citizen, where it removes any files before recreating them. We did notice a very rare case where build-runfiles tries to make in-place modifications to a temporary file that it maintains. This change ensures that build-runfiles stops doing this. Closes #19241. Commit https://github.com/bazelbuild/bazel/commit/357fc5f81b483713bb7b0319a811c4585975323d PiperOrigin-RevId: 561615005 Change-Id: I8ca95c7d35df8a53af8f632b10b4a6141d180631 Co-authored-by: Ed Schouten --- src/main/tools/build-runfiles.cc | 6 ++++ src/test/shell/integration/runfiles_test.sh | 31 +++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/src/main/tools/build-runfiles.cc b/src/main/tools/build-runfiles.cc index 9a2ef97991353a..21a6b5a7efdb32 100644 --- a/src/main/tools/build-runfiles.cc +++ b/src/main/tools/build-runfiles.cc @@ -117,6 +117,12 @@ class RunfilesCreator { void ReadManifest(const std::string &manifest_file, bool allow_relative, bool use_metadata) { + // Remove file left over from previous invocation. This ensures that + // opening succeeds if the existing file is read-only. + if (unlink(temp_filename_.c_str()) != 0 && errno != ENOENT) { + PDIE("removing temporary file at '%s/%s'", output_base_.c_str(), + temp_filename_.c_str()); + } FILE *outfile = fopen(temp_filename_.c_str(), "w"); if (!outfile) { PDIE("opening '%s/%s' for writing", output_base_.c_str(), diff --git a/src/test/shell/integration/runfiles_test.sh b/src/test/shell/integration/runfiles_test.sh index f4481ce9f15aec..998e6b35112df0 100755 --- a/src/test/shell/integration/runfiles_test.sh +++ b/src/test/shell/integration/runfiles_test.sh @@ -397,4 +397,35 @@ EOF } +function test_removal_of_old_tempfiles() { + cat > BUILD << EOF +sh_binary( + name = "foo", + srcs = ["foo.sh"], +) +EOF + touch foo.sh + chmod +x foo.sh + + # Build once to create a runfiles directory. + bazel build //:foo $EXTRA_BUILD_FLAGS >&$TEST_log || fail "build failed" + + # Remove the MANIFEST file that was created by the previous build. + # Create an inaccessible file in the place where build-runfiles writes + # its temporary results. + # + # This simulates the case where the runfiles creation process is + # interrupted and leaves the temporary file behind. The temporary file + # may become read-only if it was stored in a snapshot. + rm ${PRODUCT_NAME}-bin/foo${EXT}.runfiles/MANIFEST + touch ${PRODUCT_NAME}-bin/foo${EXT}.runfiles/MANIFEST.tmp + chmod 0 ${PRODUCT_NAME}-bin/foo${EXT}.runfiles/MANIFEST.tmp + + # Even with the inaccessible temporary file in place, build-runfiles + # should complete successfully. The MANIFEST file should be recreated. + bazel build //:foo $EXTRA_BUILD_FLAGS >&$TEST_log || fail "build failed" + [[ -f ${PRODUCT_NAME}-bin/foo${EXT}.runfiles/MANIFEST ]] \ + || fail "MANIFEST file not recreated" +} + run_suite "runfiles" From 556396fb7157e3bb4306ff780aa8c72f6ee6831a Mon Sep 17 00:00:00 2001 From: Ivo List Date: Tue, 5 Sep 2023 23:29:11 +0200 Subject: [PATCH 040/125] [6.4.0] Always fail on unknown attributes (#19404) Fixes: https://github.com/bazelbuild/bazel/issues/11000 RELNOTES[INC]: Fails on unknown attributes (even when set to None) PiperOrigin-RevId: 562519157 Change-Id: If5e430c73485c8ae9661f4231692384a057f37d5 --- .../starlark/StarlarkRuleClassFunctions.java | 4 ++ .../bazel/bzlmod/BzlmodRepoRuleCreator.java | 2 +- .../build/lib/packages/PackageFactory.java | 3 + .../build/lib/packages/RuleClass.java | 17 ++++-- .../build/lib/packages/RuleFactory.java | 5 +- .../lib/packages/WorkspaceFactoryHelper.java | 4 +- .../semantics/BuildLanguageOptions.java | 13 ++++ .../google/devtools/build/lib/analysis/BUILD | 3 +- .../analysis/RuleConfiguredTargetTest.java | 60 +++++++++++++++++++ .../build/lib/packages/RuleClassTest.java | 1 + .../build/lib/packages/RuleFactoryTest.java | 5 ++ 11 files changed, 108 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkRuleClassFunctions.java b/src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkRuleClassFunctions.java index b0ab97ac757f2f..ca2f9124f58962 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkRuleClassFunctions.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkRuleClassFunctions.java @@ -91,6 +91,7 @@ import com.google.devtools.build.lib.packages.StarlarkProviderIdentifier; import com.google.devtools.build.lib.packages.TargetUtils; import com.google.devtools.build.lib.packages.Type; +import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; import com.google.devtools.build.lib.starlarkbuildapi.StarlarkRuleFunctionsApi; import com.google.devtools.build.lib.util.FileType; @@ -799,6 +800,9 @@ public Object call(StarlarkThread thread, Tuple args, Dict kwarg pkgContext.getBuilder(), ruleClass, attributeValues, + thread + .getSemantics() + .getBool(BuildLanguageOptions.INCOMPATIBLE_FAIL_ON_UNKNOWN_ATTRIBUTES), pkgContext.getEventHandler(), thread.getSemantics(), thread.getCallStack()); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleCreator.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleCreator.java index c26fe22d4a5502..221c39c512c7b4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleCreator.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleCreator.java @@ -76,7 +76,7 @@ public static Rule createRule( try { rule = RuleFactory.createAndAddRule( - packageBuilder, ruleClass, attributeValues, eventHandler, semantics, callStack); + packageBuilder, ruleClass, attributeValues, true, eventHandler, semantics, callStack); } catch (NameConflictException e) { // This literally cannot happen -- we just created the package! throw new IllegalStateException(e); diff --git a/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java index 4b9522da2c077a..8a87df886468e5 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java +++ b/src/main/java/com/google/devtools/build/lib/packages/PackageFactory.java @@ -414,6 +414,9 @@ public NoneType call(StarlarkThread thread, Tuple args, Dict kwa context.pkgBuilder, ruleClass, new BuildLangTypedAttributeValuesMap(kwargs), + thread + .getSemantics() + .getBool(BuildLanguageOptions.INCOMPATIBLE_FAIL_ON_UNKNOWN_ATTRIBUTES), context.eventHandler, thread.getSemantics(), thread.getCallStack()); diff --git a/src/main/java/com/google/devtools/build/lib/packages/RuleClass.java b/src/main/java/com/google/devtools/build/lib/packages/RuleClass.java index 5522a6e30028fd..4b1f878b58545b 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/RuleClass.java +++ b/src/main/java/com/google/devtools/build/lib/packages/RuleClass.java @@ -2053,6 +2053,7 @@ Rule createRule( Package.Builder pkgBuilder, Label ruleLabel, AttributeValues attributeValues, + boolean failOnUnknownAttributes, EventHandler eventHandler, Location location, List callstack) @@ -2060,7 +2061,7 @@ Rule createRule( Rule rule = pkgBuilder.createRule( ruleLabel, this, location, callstack, AttributeContainer.newMutableInstance(this)); - populateRuleAttributeValues(rule, pkgBuilder, attributeValues, eventHandler); + populateRuleAttributeValues(rule, pkgBuilder, attributeValues, failOnUnknownAttributes, eventHandler); checkAspectAllowedValues(rule, eventHandler); rule.populateOutputFiles(eventHandler, pkgBuilder); checkForDuplicateLabels(rule, eventHandler); @@ -2085,7 +2086,7 @@ Rule createRuleUnchecked( throws InterruptedException, CannotPrecomputeDefaultsException { Rule rule = pkgBuilder.createRule(ruleLabel, this, location, callstack, implicitOutputsFunction); - populateRuleAttributeValues(rule, pkgBuilder, attributeValues, NullEventHandler.INSTANCE); + populateRuleAttributeValues(rule, pkgBuilder, attributeValues, true, NullEventHandler.INSTANCE); rule.populateOutputFilesUnchecked(pkgBuilder); return rule; } @@ -2101,6 +2102,7 @@ private void populateRuleAttributeValues( Rule rule, Package.Builder pkgBuilder, AttributeValues attributeValues, + boolean failOnUnknownAttributes, EventHandler eventHandler) throws InterruptedException, CannotPrecomputeDefaultsException { @@ -2109,6 +2111,7 @@ private void populateRuleAttributeValues( rule, pkgBuilder.getLabelConverter(), attributeValues, + failOnUnknownAttributes, pkgBuilder.getListInterner(), eventHandler); populateDefaultRuleAttributeValues(rule, pkgBuilder, definedAttrIndices, eventHandler); @@ -2131,6 +2134,7 @@ private BitSet populateDefinedRuleAttributeValues( Rule rule, LabelConverter labelConverter, AttributeValues attributeValues, + boolean failOnUnknownAttributes, Interner> listInterner, EventHandler eventHandler) { BitSet definedAttrIndices = new BitSet(); @@ -2138,7 +2142,7 @@ private BitSet populateDefinedRuleAttributeValues( String attributeName = attributeValues.getName(attributeAccessor); Object attributeValue = attributeValues.getValue(attributeAccessor); // Ignore all None values. - if (attributeValue == Starlark.NONE) { + if (attributeValue == Starlark.NONE && !failOnUnknownAttributes) { continue; } @@ -2160,10 +2164,15 @@ private BitSet populateDefinedRuleAttributeValues( eventHandler); continue; } + // Ignore all None values (after reporting an error) + if (attributeValue == Starlark.NONE) { + continue; + } + Attribute attr = getAttribute(attrIndex); if (attributeName.equals("licenses") && ignoreLicenses) { - rule.setAttributeValue(attr, License.NO_LICENSE, /*explicit=*/ false); + rule.setAttributeValue(attr, License.NO_LICENSE, /* explicit= */ false); definedAttrIndices.set(attrIndex); continue; } diff --git a/src/main/java/com/google/devtools/build/lib/packages/RuleFactory.java b/src/main/java/com/google/devtools/build/lib/packages/RuleFactory.java index 83d6540528f00e..1d3bb2e076f1b0 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/RuleFactory.java +++ b/src/main/java/com/google/devtools/build/lib/packages/RuleFactory.java @@ -69,6 +69,7 @@ public static Rule createRule( Package.Builder pkgBuilder, RuleClass ruleClass, BuildLangTypedAttributeValuesMap attributeValues, + boolean failOnUnknownAttributes, EventHandler eventHandler, StarlarkSemantics semantics, ImmutableList callstack) @@ -111,6 +112,7 @@ public static Rule createRule( pkgBuilder, label, generator.attributes, + failOnUnknownAttributes, eventHandler, generator.location, // see b/23974287 for rationale callstack); @@ -143,12 +145,13 @@ public static Rule createAndAddRule( Package.Builder pkgBuilder, RuleClass ruleClass, BuildLangTypedAttributeValuesMap attributeValues, + boolean failOnUnknownAttributes, EventHandler eventHandler, StarlarkSemantics semantics, ImmutableList callstack) throws InvalidRuleException, NameConflictException, InterruptedException { Rule rule = - createRule(pkgBuilder, ruleClass, attributeValues, eventHandler, semantics, callstack); + createRule(pkgBuilder, ruleClass, attributeValues, failOnUnknownAttributes, eventHandler, semantics, callstack); pkgBuilder.addRule(rule); return rule; } diff --git a/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactoryHelper.java b/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactoryHelper.java index b8af1b9fa69c3f..32486a289e6d10 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactoryHelper.java +++ b/src/main/java/com/google/devtools/build/lib/packages/WorkspaceFactoryHelper.java @@ -48,7 +48,7 @@ public static Rule createAndAddRepositoryRule( StoredEventHandler eventHandler = new StoredEventHandler(); BuildLangTypedAttributeValuesMap attributeValues = new BuildLangTypedAttributeValuesMap(kwargs); Rule rule = - RuleFactory.createRule(pkg, ruleClass, attributeValues, eventHandler, semantics, callstack); + RuleFactory.createRule(pkg, ruleClass, attributeValues, true, eventHandler, semantics, callstack); pkg.addEvents(eventHandler.getEvents()); pkg.addPosts(eventHandler.getPosts()); overwriteRule(pkg, rule); @@ -166,7 +166,7 @@ static void addBindRule( BuildLangTypedAttributeValuesMap attributeValues = new BuildLangTypedAttributeValuesMap(attributes); Rule rule = - RuleFactory.createRule(pkg, bindRuleClass, attributeValues, handler, semantics, callstack); + RuleFactory.createRule(pkg, bindRuleClass, attributeValues, true, handler, semantics, callstack); overwriteRule(pkg, rule); rule.setVisibility(ConstantRuleVisibility.PUBLIC); } diff --git a/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java b/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java index 4e3119916db530..cb43c7cf6a773a 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java +++ b/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java @@ -652,6 +652,16 @@ public final class BuildLanguageOptions extends OptionsBase { + " specified through features configuration.") public boolean experimentalGetFixedConfiguredEnvironment; + // cleanup, flip, remove after Bazel LTS in Nov 2023 + @Option( + name = "incompatible_fail_on_unknown_attributes", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.STARLARK_SEMANTICS, + effectTags = {OptionEffectTag.LOADING_AND_ANALYSIS}, + metadataTags = {OptionMetadataTag.INCOMPATIBLE_CHANGE}, + help = "If enabled, targets that have unknown attributes set to None fail.") + public boolean incompatibleFailOnUnknownAttributes; + /** * An interner to reduce the number of StarlarkSemantics instances. A single Blaze instance should * never accumulate a large number of these and being able to shortcut on object identity makes a @@ -743,6 +753,7 @@ public StarlarkSemantics toStarlarkSemantics() { .setBool( EXPERIMENTAL_GET_FIXED_CONFIGURED_ACTION_ENV, experimentalGetFixedConfiguredEnvironment) + .setBool(INCOMPATIBLE_FAIL_ON_UNKNOWN_ATTRIBUTES, incompatibleFailOnUnknownAttributes) .build(); return INTERNER.intern(semantics); } @@ -831,6 +842,8 @@ public StarlarkSemantics toStarlarkSemantics() { "-incompatible_disable_starlark_host_transitions"; public static final String EXPERIMENTAL_GET_FIXED_CONFIGURED_ACTION_ENV = "-experimental_get_fixed_configured_action_env"; + public static final String INCOMPATIBLE_FAIL_ON_UNKNOWN_ATTRIBUTES = + "-incompatible_fail_on_unknown_attributes"; // non-booleans public static final StarlarkSemantics.Key EXPERIMENTAL_BUILTINS_BZL_PATH = diff --git a/src/test/java/com/google/devtools/build/lib/analysis/BUILD b/src/test/java/com/google/devtools/build/lib/analysis/BUILD index d89744dfd73bd8..00e004beaaa340 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/BUILD +++ b/src/test/java/com/google/devtools/build/lib/analysis/BUILD @@ -9,13 +9,13 @@ filegroup( name = "srcs", testonly = 0, srcs = glob(["**"]) + [ + "//src/test/java/com/google/devtools/build/lib/analysis/allowlisting:srcs", "//src/test/java/com/google/devtools/build/lib/analysis/extra:srcs", "//src/test/java/com/google/devtools/build/lib/analysis/mock:srcs", "//src/test/java/com/google/devtools/build/lib/analysis/platform:srcs", "//src/test/java/com/google/devtools/build/lib/analysis/starlark/annotations/processor:srcs", "//src/test/java/com/google/devtools/build/lib/analysis/testing:srcs", "//src/test/java/com/google/devtools/build/lib/analysis/util:srcs", - "//src/test/java/com/google/devtools/build/lib/analysis/allowlisting:srcs", ], visibility = ["//src:__subpackages__"], ) @@ -348,6 +348,7 @@ java_test( "//src/main/java/com/google/devtools/build/lib/analysis:config/config_matching_provider", "//src/main/java/com/google/devtools/build/lib/analysis:configured_target", "//src/main/java/com/google/devtools/build/lib/analysis:required_config_fragments_provider", + "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", "//src/test/java/com/google/devtools/build/lib/analysis/util", "//third_party:guava", diff --git a/src/test/java/com/google/devtools/build/lib/analysis/RuleConfiguredTargetTest.java b/src/test/java/com/google/devtools/build/lib/analysis/RuleConfiguredTargetTest.java index da4a034cd4cf18..6bf2de28f7e0f0 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/RuleConfiguredTargetTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/RuleConfiguredTargetTest.java @@ -17,7 +17,9 @@ import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.analysis.configuredtargets.RuleConfiguredTarget; import com.google.devtools.build.lib.analysis.util.BuildViewTestCase; +import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.events.Event; import org.junit.Test; import org.junit.runner.RunWith; @@ -435,4 +437,62 @@ public void testRulesDontProvideRequiredFragmentsByDefault() throws Exception { assertThat(getConfiguredTarget("//a:config").getProvider(RequiredConfigFragmentsProvider.class)) .isNull(); } + + @Test + public void testNativeRuleAttrSetToNoneFails() throws Exception { + setBuildLanguageOptions("--incompatible_fail_on_unknown_attributes"); + scratch.file( + "p/BUILD", // + "genrule(name = 'genrule', srcs = ['a.java'], outs = ['b'], cmd = '', bat = None)"); + + reporter.removeHandler(failFastHandler); + getTarget("//p:genrule"); + + assertContainsEvent("no such attribute 'bat' in 'genrule' rule"); + } + + @Test + public void testNativeRuleAttrSetToNoneDoesntFails() throws Exception { + setBuildLanguageOptions("--noincompatible_fail_on_unknown_attributes"); + scratch.file( + "p/BUILD", // + "genrule(name = 'genrule', srcs = ['a.java'], outs = ['b'], cmd = '', bat = None)"); + + getTarget("//p:genrule"); + } + + @Test + public void testStarlarkRuleAttrSetToNoneFails() throws Exception { + setBuildLanguageOptions("--incompatible_fail_on_unknown_attributes"); + scratch.file( + "p/rule.bzl", // + "def _impl(ctx):", + " pass", + "my_rule = rule(_impl)"); + scratch.file( + "p/BUILD", // + "load(':rule.bzl', 'my_rule')", + "my_rule(name = 'my_target', bat = None)"); + + reporter.removeHandler(failFastHandler); + getTarget("//p:my_target"); + + assertContainsEvent("no such attribute 'bat' in 'my_rule' rule"); + } + + @Test + public void testStarlarkRuleAttrSetToNoneDoesntFail() throws Exception { + setBuildLanguageOptions("--noincompatible_fail_on_unknown_attributes"); + scratch.file( + "p/rule.bzl", // + "def _impl(ctx):", + " pass", + "my_rule = rule(_impl)"); + scratch.file( + "p/BUILD", // + "load(':rule.bzl', 'my_rule')", + "my_rule(name = 'my_target', bat = None)"); + + getTarget("//p:my_target"); + } } diff --git a/src/test/java/com/google/devtools/build/lib/packages/RuleClassTest.java b/src/test/java/com/google/devtools/build/lib/packages/RuleClassTest.java index b174f97486abe3..12657c66b7bf5f 100644 --- a/src/test/java/com/google/devtools/build/lib/packages/RuleClassTest.java +++ b/src/test/java/com/google/devtools/build/lib/packages/RuleClassTest.java @@ -958,6 +958,7 @@ private Rule createRule( pkgBuilder, ruleLabel, new BuildLangTypedAttributeValuesMap(attributeValues), + true, reporter, location, callstack); diff --git a/src/test/java/com/google/devtools/build/lib/packages/RuleFactoryTest.java b/src/test/java/com/google/devtools/build/lib/packages/RuleFactoryTest.java index ba68d615e0de53..be8f33f648723a 100644 --- a/src/test/java/com/google/devtools/build/lib/packages/RuleFactoryTest.java +++ b/src/test/java/com/google/devtools/build/lib/packages/RuleFactoryTest.java @@ -84,6 +84,7 @@ public void testCreateRule() throws Exception { pkgBuilder, ruleClass, new BuildLangTypedAttributeValuesMap(attributeValues), + true, new Reporter(new EventBus()), StarlarkSemantics.DEFAULT, DUMMY_STACK); @@ -144,6 +145,7 @@ public void testCreateWorkspaceRule() throws Exception { pkgBuilder, ruleClass, new BuildLangTypedAttributeValuesMap(attributeValues), + true, new Reporter(new EventBus()), StarlarkSemantics.DEFAULT, DUMMY_STACK); @@ -168,6 +170,7 @@ public void testWorkspaceRuleFailsInBuildFile() throws Exception { pkgBuilder, ruleClass, new BuildLangTypedAttributeValuesMap(attributeValues), + true, new Reporter(new EventBus()), StarlarkSemantics.DEFAULT, DUMMY_STACK)); @@ -192,6 +195,7 @@ public void testBuildRuleFailsInWorkspaceFile() throws Exception { pkgBuilder, ruleClass, new BuildLangTypedAttributeValuesMap(attributeValues), + true, new Reporter(new EventBus()), StarlarkSemantics.DEFAULT, DUMMY_STACK)); @@ -228,6 +232,7 @@ public void testOutputFileNotEqualDot() throws Exception { pkgBuilder, ruleClass, new BuildLangTypedAttributeValuesMap(attributeValues), + true, new Reporter(new EventBus()), StarlarkSemantics.DEFAULT, DUMMY_STACK)); From a9f196ac66011036c4c6fe8de4263b72340dfdd4 Mon Sep 17 00:00:00 2001 From: "bazel.build machine account" <15028808+bazel-io@users.noreply.github.com> Date: Wed, 6 Sep 2023 09:21:22 +0200 Subject: [PATCH 041/125] [6.4.0] Ignore Starlark options on commands with `allowResidue = False` (#19417) Ensures that `bazel version` does not fail when a `common` line in `.bazelrc` contains a Starlark option (similar for `sync` and `shutdown`). There is very little chance for users being confused about these commands accepting and silently ignoring Starlark flags. Fixes https://github.com/bazelbuild/bazel/pull/18130#issuecomment-1697620075 Closes #19363. Commit https://github.com/bazelbuild/bazel/commit/954b4d9b1e7fd9b0a09cd9eed2a057e43ce3f8eb PiperOrigin-RevId: 562959337 Change-Id: Ifb4f44d63f1801f6c5a8c2f421138b4014c49a06 Co-authored-by: Fabian Meumertzheim --- .../devtools/common/options/OptionsParser.java | 6 ++---- .../lib/runtime/BlazeOptionHandlerTest.java | 18 ------------------ src/test/py/bazel/options_test.py | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/google/devtools/common/options/OptionsParser.java b/src/main/java/com/google/devtools/common/options/OptionsParser.java index 0c0d2ab8dc858c..61486ae23171d7 100644 --- a/src/main/java/com/google/devtools/common/options/OptionsParser.java +++ b/src/main/java/com/google/devtools/common/options/OptionsParser.java @@ -758,10 +758,8 @@ public ImmutableList parseArgsAsExpansionOfOption( private void addResidueFromResult(OptionsParserImplResult result) throws OptionsParsingException { residue.addAll(result.getResidue()); postDoubleDashResidue.addAll(result.postDoubleDashResidue); - if (!allowResidue && (!getSkippedArgs().isEmpty() || !residue.isEmpty())) { - String errorMsg = - "Unrecognized arguments: " - + Joiner.on(' ').join(Iterables.concat(getSkippedArgs(), residue)); + if (!allowResidue && !residue.isEmpty()) { + String errorMsg = "Unrecognized arguments: " + Joiner.on(' ').join(residue); throw new OptionsParsingException(errorMsg); } } diff --git a/src/test/java/com/google/devtools/build/lib/runtime/BlazeOptionHandlerTest.java b/src/test/java/com/google/devtools/build/lib/runtime/BlazeOptionHandlerTest.java index 5382931a0a7f45..e85cef48b36ee8 100644 --- a/src/test/java/com/google/devtools/build/lib/runtime/BlazeOptionHandlerTest.java +++ b/src/test/java/com/google/devtools/build/lib/runtime/BlazeOptionHandlerTest.java @@ -437,24 +437,6 @@ public void testParseOptions_residue() { assertThat(optionHandler.getRcfileNotes()).isEmpty(); } - @Test - public void testParseOptions_disallowResidue_skippedArgsLeadToFailure() throws Exception { - ImmutableList> optionsClasses = - ImmutableList.of(TestOptions.class, CommonCommandOptions.class, ClientOptions.class); - - BlazeOptionHandlerTestHelper helper = - new BlazeOptionHandlerTestHelper( - optionsClasses, - /* allowResidue= */ false, - /* aliasFlag= */ null, - /* skipStarlarkPrefixes= */ true); - OptionsParser parser = helper.getOptionsParser(); - - OptionsParsingException e = - assertThrows(OptionsParsingException.class, () -> parser.parse("--//f=1")); - assertThat(e).hasMessageThat().isEqualTo("Unrecognized arguments: --//f=1"); - } - @Test public void testParseOptions_explicitOption() { optionHandler.parseOptions( diff --git a/src/test/py/bazel/options_test.py b/src/test/py/bazel/options_test.py index 932931ec39c4ca..9406db1dafe16f 100644 --- a/src/test/py/bazel/options_test.py +++ b/src/test/py/bazel/options_test.py @@ -269,6 +269,20 @@ def testCommonPseudoCommand_unsupportedOptionValue(self): stderr, ) + def testCommonPseudoCommand_allowResidueFalseCommandIgnoresStarlarkOptions( + self, + ): + self.ScratchFile("WORKSPACE.bazel") + self.ScratchFile( + ".bazelrc", + [ + "common --@foo//bar:flag", + ], + ) + + # Check that version doesn't fail. + self.RunBazel(["version"]) + if __name__ == "__main__": unittest.main() From d81a8c7bc2bc1795e58a07f176aaf11913af0267 Mon Sep 17 00:00:00 2001 From: kotlaja Date: Wed, 6 Sep 2023 14:58:01 +0200 Subject: [PATCH 042/125] [6.4.0] Separate PackageSpecificationProvider from its target (PackageGroupConfiguredTarget) (#19420) This is the first step towards implementing a public Starlark API for checking if a target exists in an allowlist. This feature already exist in Bazel at Head ([link](https://github.com/bazelbuild/bazel/commit/a9c9f6cb4708ce579faf0172a0d32cfc530fee2e)), but it needs to be cherrypicked for Bazel 6.4. release. Cherry picking https://github.com/bazelbuild/bazel/commit/a9c9f6cb4708ce579faf0172a0d32cfc530fee2e Co-authored-by: Yun Peng --- .../build/lib/analysis/Allowlist.java | 7 +- .../google/devtools/build/lib/analysis/BUILD | 12 +-- .../lib/analysis/ConfiguredTargetFactory.java | 2 +- .../PackageSpecificationProvider.java | 71 +++++++++++++++- .../TransitiveInfoProviderMapBuilder.java | 6 +- .../PackageGroupConfiguredTarget.java | 80 ++++--------------- .../lib/analysis/test/TestActionBuilder.java | 20 +---- .../build/lib/bazel/rules/CcRules.java | 3 + .../google/devtools/build/lib/rules/cpp/BUILD | 1 - .../devtools/build/lib/rules/java/BUILD | 2 - .../rules/java/JavaPackageConfiguration.java | 3 +- .../java/JavaPackageConfigurationRule.java | 4 +- .../build/lib/rules/java/JavaToolchain.java | 3 +- .../lib/rules/java/JavaToolchainRule.java | 4 +- .../build/lib/analysis/util/TestAspects.java | 4 +- 15 files changed, 101 insertions(+), 121 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/analysis/Allowlist.java b/src/main/java/com/google/devtools/build/lib/analysis/Allowlist.java index d6593d15e56eff..82d4ae340756b6 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/Allowlist.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/Allowlist.java @@ -22,7 +22,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.analysis.config.ExecutionTransitionFactory; -import com.google.devtools.build.lib.analysis.configuredtargets.PackageGroupConfiguredTarget; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.packages.Attribute; @@ -51,7 +50,7 @@ public static Attribute.Builder

P getProvider(Class

provider) { if (provider == FileProvider.class) { - return provider.cast(NO_FILES); // can't fail + return provider.cast(FileProvider.EMPTY); // can't fail + } + if (provider == PackageSpecificationProvider.class) { + return provider.cast(packageSpecificationProvider); } else { return super.getProvider(provider); } @@ -81,55 +62,22 @@ public

P getProvider(Class

provider) { public PackageGroupConfiguredTarget( Label label, NestedSet visibility, - NestedSet packageSpecifications) { + TargetContext targetContext, + PackageGroup packageGroup) { super(label, null, visibility); - this.packageSpecifications = packageSpecifications; + this.packageSpecificationProvider = + PackageSpecificationProvider.create(targetContext, packageGroup); } public PackageGroupConfiguredTarget(TargetContext targetContext, PackageGroup packageGroup) { - this( - targetContext.getLabel(), - targetContext.getVisibility(), - getPackageSpecifications(targetContext, packageGroup)); - } - - private static NestedSet getPackageSpecifications( - TargetContext targetContext, PackageGroup packageGroup) { - NestedSetBuilder builder = NestedSetBuilder.stableOrder(); - for (Label label : packageGroup.getIncludes()) { - TransitiveInfoCollection include = - targetContext.findDirectPrerequisite( - label, Optional.ofNullable(targetContext.getConfiguration())); - PackageSpecificationProvider provider = - include == null ? null : include.get(PackageGroupConfiguredTarget.PROVIDER); - if (provider == null) { - targetContext - .getAnalysisEnvironment() - .getEventHandler() - .handle( - Event.error( - targetContext.getTarget().getLocation(), - String.format("label '%s' does not refer to a package group", label))); - continue; - } - - builder.addTransitive(provider.getPackageSpecifications()); - } - - builder.add(packageGroup.getPackageSpecifications()); - return builder.build(); - } - - @Override - public NestedSet getPackageSpecifications() { - return packageSpecifications; + this(targetContext.getLabel(), targetContext.getVisibility(), targetContext, packageGroup); } @Override @Nullable protected Info rawGetStarlarkProvider(Provider.Key providerKey) { - if (providerKey.equals(PROVIDER.getKey())) { - return this; + if (providerKey.equals(packageSpecificationProvider.getProvider().getKey())) { + return packageSpecificationProvider; } return null; } @@ -156,6 +104,6 @@ public boolean starlarkMatches(Label label, StarlarkThread starlarkThread) throw if (!"@_builtins".equals(repository.getNameWithAt())) { throw Starlark.errorf("private API only for use by builtins"); } - return Allowlist.isAvailableFor(getPackageSpecifications(), label); + return Allowlist.isAvailableFor(packageSpecificationProvider.getPackageSpecifications(), label); } } diff --git a/src/main/java/com/google/devtools/build/lib/analysis/test/TestActionBuilder.java b/src/main/java/com/google/devtools/build/lib/analysis/test/TestActionBuilder.java index cf908ce46c1651..dedaff8bfdf3a8 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/test/TestActionBuilder.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/test/TestActionBuilder.java @@ -40,15 +40,11 @@ import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; import com.google.devtools.build.lib.analysis.actions.LazyWriteNestedSetOfPairAction; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; -import com.google.devtools.build.lib.analysis.configuredtargets.PackageGroupConfiguredTarget; import com.google.devtools.build.lib.analysis.test.TestProvider.TestParams; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; import com.google.devtools.build.lib.collect.nestedset.Order; -import com.google.devtools.build.lib.packages.PackageSpecification.PackageGroupContents; import com.google.devtools.build.lib.packages.TestTimeout; -import com.google.devtools.build.lib.skyframe.serialization.autocodec.AutoCodec.VisibleForSerialization; -import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; import com.google.devtools.build.lib.util.OS; import com.google.devtools.build.lib.util.Pair; import com.google.devtools.build.lib.vfs.PathFragment; @@ -75,20 +71,6 @@ public final class TestActionBuilder { private static final String COVERAGE_REPORTED_TO_ACTUAL_SOURCES_FILE = "COVERAGE_REPORTED_TO_ACTUAL_SOURCES_FILE"; - static class EmptyPackageProvider extends PackageGroupConfiguredTarget { - public EmptyPackageProvider() { - super(null, null, null); - } - - @Override - public NestedSet getPackageSpecifications() { - return NestedSetBuilder.emptySet(Order.STABLE_ORDER); - } - } - - @VisibleForSerialization @SerializationConstant - static final PackageSpecificationProvider EMPTY_PACKAGES_PROVIDER = new EmptyPackageProvider(); - private final RuleContext ruleContext; private final ImmutableList.Builder additionalTools; private RunfilesSupport runfilesSupport; @@ -462,7 +444,7 @@ private TestParams createTestAction(int shards) MoreObjects.firstNonNull( Allowlist.fetchPackageSpecificationProviderOrNull( ruleContext, "external_network"), - EMPTY_PACKAGES_PROVIDER)); + PackageSpecificationProvider.EMPTY)); testOutputs.addAll(testRunnerAction.getSpawnOutputs()); testOutputs.addAll(testRunnerAction.getOutputs()); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/CcRules.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/CcRules.java index cd749903685c85..f111e3bab0c1d5 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/CcRules.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/CcRules.java @@ -16,6 +16,7 @@ import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider.RuleSet; +import com.google.devtools.build.lib.analysis.PackageSpecificationProvider; import com.google.devtools.build.lib.analysis.StaticallyLinkedMarkerProvider; import com.google.devtools.build.lib.bazel.rules.cpp.BazelCcBinaryRule; import com.google.devtools.build.lib.bazel.rules.cpp.BazelCcImportRule; @@ -93,6 +94,8 @@ public void init(ConfiguredRuleClassProvider.Builder builder) { builder.addStarlarkBuiltinsInternal( "StaticallyLinkedMarkerProvider", StaticallyLinkedMarkerProvider.PROVIDER); builder.addStarlarkBuiltinsInternal("CcNativeLibraryInfo", CcNativeLibraryInfo.PROVIDER); + builder.addStarlarkBuiltinsInternal( + "PackageSpecificationInfo", PackageSpecificationProvider.PROVIDER); builder.addStarlarkBootstrap( new CcBootstrap( new BazelCcModule(), diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/BUILD b/src/main/java/com/google/devtools/build/lib/rules/cpp/BUILD index 5701899c944fac..3f31cad14a3362 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/BUILD +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/BUILD @@ -64,7 +64,6 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:file_provider", "//src/main/java/com/google/devtools/build/lib/analysis:licenses_provider", "//src/main/java/com/google/devtools/build/lib/analysis:make_variable_supplier", - "//src/main/java/com/google/devtools/build/lib/analysis:package_specification_provider", "//src/main/java/com/google/devtools/build/lib/analysis:platform_configuration", "//src/main/java/com/google/devtools/build/lib/analysis:rule_definition_environment", "//src/main/java/com/google/devtools/build/lib/analysis:starlark/args", diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/BUILD b/src/main/java/com/google/devtools/build/lib/rules/java/BUILD index 9b46beb4a57c3c..9cb5732acab181 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/BUILD +++ b/src/main/java/com/google/devtools/build/lib/rules/java/BUILD @@ -44,7 +44,6 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:config/transitions/no_transition", "//src/main/java/com/google/devtools/build/lib/analysis:configured_target", "//src/main/java/com/google/devtools/build/lib/analysis:file_provider", - "//src/main/java/com/google/devtools/build/lib/analysis:package_specification_provider", "//src/main/java/com/google/devtools/build/lib/analysis:provider_collection", "//src/main/java/com/google/devtools/build/lib/analysis:rule_definition_environment", "//src/main/java/com/google/devtools/build/lib/analysis:template_variable_info", @@ -151,7 +150,6 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:config/invalid_configuration_exception", "//src/main/java/com/google/devtools/build/lib/analysis:config/toolchain_type_requirement", "//src/main/java/com/google/devtools/build/lib/analysis:file_provider", - "//src/main/java/com/google/devtools/build/lib/analysis:package_specification_provider", "//src/main/java/com/google/devtools/build/lib/analysis:platform_options", "//src/main/java/com/google/devtools/build/lib/analysis:provider_collection", "//src/main/java/com/google/devtools/build/lib/analysis:rule_definition_environment", diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaPackageConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPackageConfiguration.java index 2ab7c02b81daa6..804075adf486d5 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaPackageConfiguration.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPackageConfiguration.java @@ -25,7 +25,6 @@ import com.google.devtools.build.lib.analysis.RuleContext; import com.google.devtools.build.lib.analysis.Runfiles; import com.google.devtools.build.lib.analysis.RunfilesProvider; -import com.google.devtools.build.lib.analysis.configuredtargets.PackageGroupConfiguredTarget; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; import com.google.devtools.build.lib.collect.nestedset.Order; @@ -40,7 +39,7 @@ public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException, RuleErrorException, ActionConflictException { ImmutableList packages = ImmutableList.copyOf( - ruleContext.getPrerequisites("packages", PackageGroupConfiguredTarget.class)); + ruleContext.getPrerequisites("packages", PackageSpecificationProvider.class)); FileProvider dataProvider = ruleContext.getPrerequisite("data", FileProvider.class); NestedSet data = dataProvider != null diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaPackageConfigurationRule.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPackageConfigurationRule.java index 19fe1685d0ecf5..577bb8488b2b4b 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaPackageConfigurationRule.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaPackageConfigurationRule.java @@ -20,11 +20,11 @@ import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.PackageSpecificationProvider; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; import com.google.devtools.build.lib.analysis.config.ConfigAwareRuleClassBuilder; import com.google.devtools.build.lib.analysis.config.ExecutionTransitionFactory; -import com.google.devtools.build.lib.analysis.configuredtargets.PackageGroupConfiguredTarget; import com.google.devtools.build.lib.packages.RuleClass; import com.google.devtools.build.lib.packages.Type; import com.google.devtools.build.lib.util.FileTypeSet; @@ -46,7 +46,7 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment envi attr("packages", LABEL_LIST) .cfg(ExecutionTransitionFactory.create()) .allowedFileTypes() - .mandatoryProviders(ImmutableList.of(PackageGroupConfiguredTarget.PROVIDER.id()))) + .mandatoryBuiltinProviders(ImmutableList.of(PackageSpecificationProvider.class))) /* Java compiler flags. */ diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchain.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchain.java index 20db593e703e21..2d846f8e72d18f 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchain.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchain.java @@ -35,7 +35,6 @@ import com.google.devtools.build.lib.analysis.RuleContext; import com.google.devtools.build.lib.analysis.Runfiles; import com.google.devtools.build.lib.analysis.RunfilesProvider; -import com.google.devtools.build.lib.analysis.configuredtargets.PackageGroupConfiguredTarget; import com.google.devtools.build.lib.analysis.platform.ToolchainInfo; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; @@ -137,7 +136,7 @@ public ConfiguredTarget create(RuleContext ruleContext) ImmutableList jspecifyPackages = ImmutableList.copyOf( ruleContext.getPrerequisites( - "jspecify_packages", PackageGroupConfiguredTarget.class)); + "jspecify_packages", PackageSpecificationProvider.class)); jspecifyInfo = JspecifyInfo.create( jspecifyProcessor, jspecifyImplicitDeps, jspecifyJavacopts.build(), jspecifyPackages); diff --git a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainRule.java b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainRule.java index 67962dfc001ddb..d771dddb69a54c 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainRule.java +++ b/src/main/java/com/google/devtools/build/lib/rules/java/JavaToolchainRule.java @@ -25,11 +25,11 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.analysis.BaseRuleClasses; +import com.google.devtools.build.lib.analysis.PackageSpecificationProvider; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; import com.google.devtools.build.lib.analysis.config.ExecutionTransitionFactory; import com.google.devtools.build.lib.analysis.config.transitions.NoTransition; -import com.google.devtools.build.lib.analysis.configuredtargets.PackageGroupConfiguredTarget; import com.google.devtools.build.lib.packages.RuleClass; import com.google.devtools.build.lib.util.FileTypeSet; import java.util.List; @@ -363,7 +363,7 @@ The Java target version (e.g., '6' or '7'). It specifies for which Java runtime attr("jspecify_packages", LABEL_LIST) .cfg(ExecutionTransitionFactory.create()) .allowedFileTypes() - .mandatoryProviders(ImmutableList.of(PackageGroupConfiguredTarget.PROVIDER.id())) + .mandatoryBuiltinProviders(ImmutableList.of(PackageSpecificationProvider.class)) .undocumented("experimental")) .add( attr(":bytecode_optimizer", LABEL) diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/TestAspects.java b/src/test/java/com/google/devtools/build/lib/analysis/util/TestAspects.java index f31f641ac3fb9e..419d3c0e0a6358 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/TestAspects.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/TestAspects.java @@ -30,6 +30,7 @@ import com.google.devtools.build.lib.analysis.ConfiguredAspect; import com.google.devtools.build.lib.analysis.ConfiguredAspectFactory; import com.google.devtools.build.lib.analysis.ConfiguredTarget; +import com.google.devtools.build.lib.analysis.PackageSpecificationProvider; import com.google.devtools.build.lib.analysis.RuleConfiguredTargetBuilder; import com.google.devtools.build.lib.analysis.RuleConfiguredTargetFactory; import com.google.devtools.build.lib.analysis.RuleContext; @@ -37,7 +38,6 @@ import com.google.devtools.build.lib.analysis.RunfilesProvider; import com.google.devtools.build.lib.analysis.TransitiveInfoCollection; import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; -import com.google.devtools.build.lib.analysis.configuredtargets.PackageGroupConfiguredTarget; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.RepositoryName; @@ -433,7 +433,7 @@ public AspectDefinition getDefinition(AspectParameters aspectParameters) { .add( attr("$dep", LABEL) .value(Label.parseAbsoluteUnchecked("//extra:extra")) - .mandatoryProviders(ImmutableList.of(PackageGroupConfiguredTarget.PROVIDER.id()))) + .mandatoryBuiltinProviders(ImmutableList.of(PackageSpecificationProvider.class))) .build(); public static final ComputedAttributeAspect COMPUTED_ATTRIBUTE_ASPECT = From ad813e2fbd647b8636b65363412add58ee96fb98 Mon Sep 17 00:00:00 2001 From: kotlaja Date: Wed, 6 Sep 2023 17:32:02 +0200 Subject: [PATCH 043/125] [6.4.0] Expose PackageSpecificationInfo provider as a top level symbol (#19422) This is step towards implementing a public Starlark API for checking if a target exists in an allowlist. Cherry picking d1d35b2. --- .../PackageSpecificationProvider.java | 4 ++- .../bazel/rules/BazelRuleClassProvider.java | 10 ++++++++ .../build/lib/bazel/rules/CcRules.java | 3 --- .../PackageSpecificationProviderApi.java | 25 +++++++++++++++++++ 4 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 src/main/java/com/google/devtools/build/lib/starlarkbuildapi/PackageSpecificationProviderApi.java diff --git a/src/main/java/com/google/devtools/build/lib/analysis/PackageSpecificationProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/PackageSpecificationProvider.java index a6f63e8de37631..a6ce95562ae2a0 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/PackageSpecificationProvider.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/PackageSpecificationProvider.java @@ -24,13 +24,15 @@ import com.google.devtools.build.lib.packages.PackageGroup; import com.google.devtools.build.lib.packages.PackageSpecification.PackageGroupContents; import com.google.devtools.build.lib.packages.Provider; +import com.google.devtools.build.lib.starlarkbuildapi.PackageSpecificationProviderApi; import java.util.Optional; /** * A {@link TransitiveInfoProvider} that describes a set of transitive package specifications used * in package groups. */ -public class PackageSpecificationProvider extends NativeInfo implements TransitiveInfoProvider { +public class PackageSpecificationProvider extends NativeInfo + implements TransitiveInfoProvider, PackageSpecificationProviderApi { private static final String STARLARK_NAME = "PackageSpecificationInfo"; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java index 3e8c2fd7ae074c..df645a1a52e53d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/BazelRuleClassProvider.java @@ -24,6 +24,7 @@ import com.google.devtools.build.lib.actions.ActionEnvironment; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider.RuleSet; +import com.google.devtools.build.lib.analysis.PackageSpecificationProvider; import com.google.devtools.build.lib.analysis.PlatformConfiguration; import com.google.devtools.build.lib.analysis.ShellConfiguration; import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue; @@ -509,6 +510,14 @@ public ImmutableList requires() { } }; + static final RuleSet PACKAGING_RULES = + new RuleSet() { + @Override + public void init(ConfiguredRuleClassProvider.Builder builder) { + builder.addStarlarkAccessibleTopLevels("PackageSpecificationInfo", PackageSpecificationProvider.PROVIDER); + } + }; + private static final ImmutableSet RULE_SETS = ImmutableSet.of( BAZEL_SETUP, @@ -529,6 +538,7 @@ public ImmutableList requires() { J2ObjcRules.INSTANCE, TestingSupportRules.INSTANCE, VARIOUS_WORKSPACE_RULES, + PACKAGING_RULES, // This rule set is a little special: it needs to depend on every configuration fragment // that has Make variables, so we put it last. ToolchainRules.INSTANCE); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/CcRules.java b/src/main/java/com/google/devtools/build/lib/bazel/rules/CcRules.java index f111e3bab0c1d5..cd749903685c85 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/CcRules.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/CcRules.java @@ -16,7 +16,6 @@ import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider.RuleSet; -import com.google.devtools.build.lib.analysis.PackageSpecificationProvider; import com.google.devtools.build.lib.analysis.StaticallyLinkedMarkerProvider; import com.google.devtools.build.lib.bazel.rules.cpp.BazelCcBinaryRule; import com.google.devtools.build.lib.bazel.rules.cpp.BazelCcImportRule; @@ -94,8 +93,6 @@ public void init(ConfiguredRuleClassProvider.Builder builder) { builder.addStarlarkBuiltinsInternal( "StaticallyLinkedMarkerProvider", StaticallyLinkedMarkerProvider.PROVIDER); builder.addStarlarkBuiltinsInternal("CcNativeLibraryInfo", CcNativeLibraryInfo.PROVIDER); - builder.addStarlarkBuiltinsInternal( - "PackageSpecificationInfo", PackageSpecificationProvider.PROVIDER); builder.addStarlarkBootstrap( new CcBootstrap( new BazelCcModule(), diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/PackageSpecificationProviderApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/PackageSpecificationProviderApi.java new file mode 100644 index 00000000000000..c313719dcbbd5c --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/PackageSpecificationProviderApi.java @@ -0,0 +1,25 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.lib.starlarkbuildapi; + +import com.google.devtools.build.docgen.annot.DocCategory; +import com.google.devtools.build.lib.starlarkbuildapi.core.StructApi; +import net.starlark.java.annot.StarlarkBuiltin; + +/** Provider which describes a set of transitive package specifications used in package groups. */ +@StarlarkBuiltin( + name = "PackageSpecificationInfo", + doc = "Information about transitive package specifications used in package groups.", + category = DocCategory.PROVIDER) +public interface PackageSpecificationProviderApi extends StructApi {} From 4eb2ba6fe184c62e5a50a1b44e77d3ab2e3445cf Mon Sep 17 00:00:00 2001 From: "Ian (Hee) Cha" Date: Wed, 6 Sep 2023 10:50:09 -0700 Subject: [PATCH 044/125] [6.4.0] Revert "Report remote execution messages as events" (#19415) Since it makes it harder for BEP consumers to present the messages in a structured way. Fixes #19327. This reverts commit 6d089b3f399b993f296fab60966f1de9f07a3d0b. Closes #19347. Commit https://github.com/bazelbuild/bazel/commit/d3d167f3f6048d74e882b950189c06cbe234bf03 PiperOrigin-RevId: 562793125 Change-Id: I6ea41323c4ea98d416e4cfd9728d0d753aef9b5e Co-authored-by: Chi Wang --- .../build/lib/remote/RemoteSpawnRunner.java | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java index b23b585ab24e51..2c8b937ce19947 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java @@ -67,6 +67,7 @@ import com.google.devtools.build.lib.server.FailureDetails; import com.google.devtools.build.lib.server.FailureDetails.FailureDetail; import com.google.devtools.build.lib.util.ExitCode; +import com.google.devtools.build.lib.util.io.FileOutErr; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.longrunning.Operation; @@ -288,7 +289,7 @@ public SpawnResult exec(Spawn spawn, SpawnExecutionContext context) // It's already late at this stage, but we should at least report once. reporter.reportExecutingIfNot(); - maybePrintExecutionMessages(spawn, result.getMessage(), result.success()); + maybePrintExecutionMessages(context, result.getMessage(), result.success()); profileAccounting(result.getExecutionMetadata()); spawnMetricsAccounting(spawnMetrics, result.getExecutionMetadata()); @@ -459,16 +460,14 @@ public boolean handlesCaching() { return true; } - private void maybePrintExecutionMessages(Spawn spawn, String message, boolean success) { + private void maybePrintExecutionMessages( + SpawnExecutionContext context, String message, boolean success) { + FileOutErr outErr = context.getFileOutErr(); boolean printMessage = remoteOptions.remotePrintExecutionMessages.shouldPrintMessages(success) && !message.isEmpty(); if (printMessage) { - report( - Event.info( - String.format( - "Remote execution message for %s %s: %s", - spawn.getMnemonic(), spawn.getTargetLabel(), message))); + outErr.printErr(message + "\n"); } } @@ -550,8 +549,7 @@ private SpawnResult handleError( } } if (e.isExecutionTimeout()) { - maybePrintExecutionMessages( - action.getSpawn(), e.getResponse().getMessage(), /* success= */ false); + maybePrintExecutionMessages(context, e.getResponse().getMessage(), /* success= */ false); return new SpawnResult.Builder() .setRunnerName(getName()) .setStatus(Status.TIMEOUT) From 173199ccea173f2ae2d623d8bf486114cd8fd415 Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Wed, 6 Sep 2023 13:43:28 -0700 Subject: [PATCH 045/125] [6.4] Add --incompatible_disable_objc_library_transition (#19393) This is a migration for https://github.com/bazelbuild/bazel/issues/16870 Users who rely on the current behavior should instead wrap their library in a target from [rules_apple](https://github.com/bazelbuild/rules_apple). Closes https://github.com/bazelbuild/bazel/issues/19383. PiperOrigin-RevId: 561253613 Change-Id: I1b1b24a08caa33e86881bfe376a525060c9887a9 (cherry picked from commit 0f34e76f45a2d9a6407431ea3a371a9d62814527) --- .../semantics/BuildLanguageOptions.java | 16 +++++++++++++ .../build/lib/rules/cpp/CcModule.java | 8 +++++++ .../lib/starlarkbuildapi/cpp/CcModuleApi.java | 6 +++++ .../builtins_bzl/common/cc/semantics.bzl | 4 ++++ .../builtins_bzl/common/objc/objc_library.bzl | 3 ++- .../build/lib/rules/objc/ObjcLibraryTest.java | 24 +++++++++++++++++++ 6 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java b/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java index cb43c7cf6a773a..2259ab5a68e7d5 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java +++ b/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java @@ -662,6 +662,17 @@ public final class BuildLanguageOptions extends OptionsBase { help = "If enabled, targets that have unknown attributes set to None fail.") public boolean incompatibleFailOnUnknownAttributes; + @Option( + name = "incompatible_disable_objc_library_transition", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.STARLARK_SEMANTICS, + effectTags = {OptionEffectTag.BUILD_FILE_SEMANTICS}, + metadataTags = {OptionMetadataTag.INCOMPATIBLE_CHANGE}, + help = + "Disable objc_library's custom transition and inherit " + + "from the top level target instead") + public boolean incompatibleDisableObjcLibraryTransition; + /** * An interner to reduce the number of StarlarkSemantics instances. A single Blaze instance should * never accumulate a large number of these and being able to shortcut on object identity makes a @@ -754,6 +765,9 @@ public StarlarkSemantics toStarlarkSemantics() { EXPERIMENTAL_GET_FIXED_CONFIGURED_ACTION_ENV, experimentalGetFixedConfiguredEnvironment) .setBool(INCOMPATIBLE_FAIL_ON_UNKNOWN_ATTRIBUTES, incompatibleFailOnUnknownAttributes) + .setBool( + INCOMPATIBLE_DISABLE_OBJC_LIBRARY_TRANSITION, + incompatibleDisableObjcLibraryTransition) .build(); return INTERNER.intern(semantics); } @@ -844,6 +858,8 @@ public StarlarkSemantics toStarlarkSemantics() { "-experimental_get_fixed_configured_action_env"; public static final String INCOMPATIBLE_FAIL_ON_UNKNOWN_ATTRIBUTES = "-incompatible_fail_on_unknown_attributes"; + public static final String INCOMPATIBLE_DISABLE_OBJC_LIBRARY_TRANSITION = + "-incompatible_disable_objc_library_transition"; // non-booleans public static final StarlarkSemantics.Key EXPERIMENTAL_BUILTINS_BZL_PATH = diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java index 5065e3169627bf..22a575ca1f4bcb 100755 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CcModule.java @@ -1113,6 +1113,14 @@ public void checkExperimentalStarlarkCcImport(StarlarkActionFactory starlarkActi } } + @Override + public boolean getIncompatibleDisableObjcLibraryTransition(StarlarkThread thread) + throws EvalException { + return thread + .getSemantics() + .getBool(BuildLanguageOptions.INCOMPATIBLE_DISABLE_OBJC_LIBRARY_TRANSITION); + } + @Override public CcLinkingContext createCcLinkingInfo( Object linkerInputs, diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/cpp/CcModuleApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/cpp/CcModuleApi.java index b7a91401ff9858..6489080a0c904d 100755 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/cpp/CcModuleApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/cpp/CcModuleApi.java @@ -777,6 +777,12 @@ LinkerInputT createLinkerInput( void checkExperimentalStarlarkCcImport(StarlarkActionFactoryT starlarkActionFactoryApi) throws EvalException; + @StarlarkMethod( + name = "incompatible_disable_objc_library_transition", + useStarlarkThread = true, + documented = false) + boolean getIncompatibleDisableObjcLibraryTransition(StarlarkThread thread) throws EvalException; + @StarlarkMethod( name = "create_linking_context", doc = "Creates a LinkingContext.", diff --git a/src/main/starlark/builtins_bzl/common/cc/semantics.bzl b/src/main/starlark/builtins_bzl/common/cc/semantics.bzl index cedeed5b282b5c..14cb4f4c422404 100644 --- a/src/main/starlark/builtins_bzl/common/cc/semantics.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/semantics.bzl @@ -144,6 +144,9 @@ def _check_experimental_cc_shared_library(ctx): if not cc_common.check_experimental_cc_shared_library(): fail("Pass --experimental_cc_shared_library to use cc_shared_library") +def _incompatible_disable_objc_library_transition(): + return cc_common.incompatible_disable_objc_library_transition() + def _get_linkstatic_default(ctx): if ctx.attr._is_test: # By default Tests do not link statically. Except on Windows. @@ -194,4 +197,5 @@ semantics = struct( get_coverage_attrs = _get_coverage_attrs, get_coverage_env = _get_coverage_env, get_proto_aspects = _get_proto_aspects, + incompatible_disable_objc_library_transition = _incompatible_disable_objc_library_transition, ) diff --git a/src/main/starlark/builtins_bzl/common/objc/objc_library.bzl b/src/main/starlark/builtins_bzl/common/objc/objc_library.bzl index 47f5377ae0ed27..081b4e66347b84 100644 --- a/src/main/starlark/builtins_bzl/common/objc/objc_library.bzl +++ b/src/main/starlark/builtins_bzl/common/objc/objc_library.bzl @@ -18,6 +18,7 @@ load("@_builtins//:common/objc/compilation_support.bzl", "compilation_support") load("@_builtins//:common/objc/attrs.bzl", "common_attrs") load("@_builtins//:common/objc/transitions.bzl", "apple_crosstool_transition") load("@_builtins//:common/cc/cc_helper.bzl", "cc_helper") +load("@_builtins//:common/cc/semantics.bzl", "semantics") objc_internal = _builtins.internal.objc_internal CcInfo = _builtins.toplevel.CcInfo @@ -102,7 +103,7 @@ objc_library = rule( common_attrs.XCRUN_RULE, ), fragments = ["objc", "apple", "cpp"], - cfg = apple_crosstool_transition, + cfg = None if semantics.incompatible_disable_objc_library_transition() else apple_crosstool_transition, toolchains = cc_helper.use_cpp_toolchain(), incompatible_use_toolchain_transition = True, ) diff --git a/src/test/java/com/google/devtools/build/lib/rules/objc/ObjcLibraryTest.java b/src/test/java/com/google/devtools/build/lib/rules/objc/ObjcLibraryTest.java index 80d06e2dd1bcab..0856da8c74d604 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/objc/ObjcLibraryTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/objc/ObjcLibraryTest.java @@ -109,6 +109,30 @@ public void testConfigTransitionWithTopLevelAppleConfiguration() throws Exceptio assertThat(objcObject.getExecPathString()).contains("ios_x86_64"); } + @Test + public void testNoTransition() throws Exception { + scratch.file( + "bin/BUILD", + "objc_library(", + " name = 'objc',", + " srcs = ['objc.m'],", + ")", + "cc_binary(", + " name = 'cc',", + " srcs = ['cc.cc'],", + " deps = [':objc'],", + ")"); + + setBuildLanguageOptions("--incompatible_disable_objc_library_transition"); + useConfiguration("--macos_cpus=arm64,x86_64"); + + ConfiguredTarget cc = getConfiguredTarget("//bin:cc"); + Artifact objcObject = + ActionsTestUtil.getFirstArtifactEndingWith( + actionsTestUtil().artifactClosureOf(getFilesToBuild(cc)), "objc.o"); + assertThat(objcObject.getExecPathString()).contains("k8-fastbuild"); + } + @Test public void testFilesToBuild() throws Exception { ConfiguredTarget target = From eea0909c0bff29d2e8fe34a6be4ac14f0dd5f369 Mon Sep 17 00:00:00 2001 From: "Ian (Hee) Cha" Date: Wed, 6 Sep 2023 15:12:40 -0700 Subject: [PATCH 046/125] [6.4.0] Create .bazelversion to address postsubmit timeout issues (#19435) --- .bazelversion | 1 + BUILD | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 .bazelversion diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 00000000000000..0df17dd0f6a311 --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +6.2.1 \ No newline at end of file diff --git a/BUILD b/BUILD index 6a3a2272fb39a5..6e7fc3450c2604 100644 --- a/BUILD +++ b/BUILD @@ -40,7 +40,10 @@ filegroup( "//third_party:srcs", "//src/main/starlark/tests/builtins_bzl:srcs", "//src/main/java/com/google/devtools/build/docgen/release:srcs", - ] + glob([".bazelci/*"]) + [".bazelrc"], + ] + glob([".bazelci/*"]) + [ + ".bazelrc", + ".bazelversion", + ], applicable_licenses = ["@io_bazel//:license"], visibility = ["//src/test/shell/bazel:__pkg__"], ) From c4d5fdb38b0f8e38c72106a60b40dda142a4dfe8 Mon Sep 17 00:00:00 2001 From: kotlaja Date: Thu, 7 Sep 2023 13:29:14 +0200 Subject: [PATCH 047/125] [6.4.0] Add `contains` method inside `PackageSpecificationProvider` (#19425) This method checks if a target exists inside an allowlist. Cherry picking 8828015. --- .../PackageSpecificationProvider.java | 18 +++ .../PackageSpecificationProviderApi.java | 20 ++- .../analysis/allowlisting/AllowlistTest.java | 152 ++++++++++++++++++ 3 files changed, 189 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/analysis/PackageSpecificationProvider.java b/src/main/java/com/google/devtools/build/lib/analysis/PackageSpecificationProvider.java index a6ce95562ae2a0..72b778ad2803f3 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/PackageSpecificationProvider.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/PackageSpecificationProvider.java @@ -15,6 +15,7 @@ package com.google.devtools.build.lib.analysis; import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.collect.nestedset.NestedSet; import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder; import com.google.devtools.build.lib.collect.nestedset.Order; @@ -26,6 +27,8 @@ import com.google.devtools.build.lib.packages.Provider; import com.google.devtools.build.lib.starlarkbuildapi.PackageSpecificationProviderApi; import java.util.Optional; +import net.starlark.java.eval.EvalException; +import net.starlark.java.eval.Starlark; /** * A {@link TransitiveInfoProvider} that describes a set of transitive package specifications used @@ -92,4 +95,19 @@ private static NestedSet getPackageSpecifications( builder.add(packageGroup.getPackageSpecifications()); return builder.build(); } + + @Override + public boolean targetInAllowlist(Object target) throws EvalException, LabelSyntaxException { + Label targetLabel; + if (target instanceof String) { + targetLabel = Label.parseCanonical((String) target); + } else if (target instanceof Label) { + targetLabel = (Label) target; + } else { + throw Starlark.errorf( + "expected string or label for 'target' instead of %s", Starlark.type(target)); + } + + return Allowlist.isAvailableFor(packageSpecifications, targetLabel); + } } diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/PackageSpecificationProviderApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/PackageSpecificationProviderApi.java index c313719dcbbd5c..c81eb49665d217 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/PackageSpecificationProviderApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/PackageSpecificationProviderApi.java @@ -14,12 +14,30 @@ package com.google.devtools.build.lib.starlarkbuildapi; import com.google.devtools.build.docgen.annot.DocCategory; +import com.google.devtools.build.lib.cmdline.Label; +import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.starlarkbuildapi.core.StructApi; +import net.starlark.java.annot.Param; +import net.starlark.java.annot.ParamType; import net.starlark.java.annot.StarlarkBuiltin; +import net.starlark.java.annot.StarlarkMethod; +import net.starlark.java.eval.EvalException; /** Provider which describes a set of transitive package specifications used in package groups. */ @StarlarkBuiltin( name = "PackageSpecificationInfo", doc = "Information about transitive package specifications used in package groups.", category = DocCategory.PROVIDER) -public interface PackageSpecificationProviderApi extends StructApi {} +public interface PackageSpecificationProviderApi extends StructApi { + @StarlarkMethod( + name = "contains", + doc = "Checks if a target exists in a package group.", + parameters = { + @Param( + name = "target", + positional = true, + doc = "A target which is checked if it exists inside the package group.", + allowedTypes = {@ParamType(type = Label.class), @ParamType(type = String.class)}) + }) + public boolean targetInAllowlist(Object target) throws EvalException, LabelSyntaxException; +} diff --git a/src/test/java/com/google/devtools/build/lib/analysis/allowlisting/AllowlistTest.java b/src/test/java/com/google/devtools/build/lib/analysis/allowlisting/AllowlistTest.java index 828bbe7b343580..d65485406ee7ab 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/allowlisting/AllowlistTest.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/allowlisting/AllowlistTest.java @@ -125,4 +125,156 @@ public void testIncludes() throws Exception { getConfiguredTarget("//x:x"); assertNoEvents(); } + + @Test + public void targetInAllowlist_targetAsStringParameter() throws Exception { + scratch.file( + "allowlist/BUILD", + "package_group(", + " name='allowlist',", + " packages=[", + " '//direct',", + " ])"); + scratch.file( + "test/rule.bzl", + "def _impl(ctx):", + " target = '//direct:rule_from_allowlist'", + " target_in_allowlist =" + + " ctx.attr._allowlist_test[PackageSpecificationInfo].contains(target)", + " if not target_in_allowlist:", + " fail('Target should be in the allowlist')", + " return []", + "custom_rule = rule(", + " implementation = _impl,", + " attrs = {", + " '_allowlist_test': attr.label(", + " default = '//allowlist:allowlist',", + " cfg = 'exec',", + " providers = ['PackageSpecificationInfo']", + " ),", + " },", + ")"); + scratch.file( + "test/BUILD", + "load('//test:rule.bzl', 'custom_rule')", + "custom_rule(name='allowlist_rule')"); + + getConfiguredTarget("//test:allowlist_rule"); + + assertNoEvents(); + } + + @Test + public void targetInAllowlist_targetAsLabelParameter() throws Exception { + scratch.file( + "allowlist/BUILD", + "package_group(", + " name='allowlist',", + " packages=[", + " '//test',", + " ])"); + scratch.file( + "test/rule.bzl", + "def _impl(ctx):", + " target = ctx.label", + " target_in_allowlist =" + + " ctx.attr._allowlist_test[PackageSpecificationInfo].contains(target)", + " if not target_in_allowlist:", + " fail('Target should be in the allowlist')", + " return []", + "custom_rule = rule(", + " implementation = _impl,", + " attrs = {", + " '_allowlist_test': attr.label(", + " default = '//allowlist:allowlist',", + " cfg = 'exec',", + " providers = [PackageSpecificationInfo]", + " ),", + " },", + ")"); + scratch.file( + "test/BUILD", + "load('//test:rule.bzl', 'custom_rule')", + "custom_rule(name='allowlist_rule')"); + + getConfiguredTarget("//test:allowlist_rule"); + + assertNoEvents(); + } + + @Test + public void targetNotInAllowlist() throws Exception { + scratch.file( + "allowlist/BUILD", + "package_group(", + " name='allowlist',", + " packages=[", + " '//direct',", + " ])"); + scratch.file( + "test/rule.bzl", + "def _impl(ctx):", + " target = '//non_direct:rule_not_from_allowlist'", + " target_in_allowlist =" + + " ctx.attr._allowlist_test[PackageSpecificationInfo].contains(target)", + " if target_in_allowlist:", + " fail('Target should not be in the allowlist')", + " return []", + "custom_rule = rule(", + " implementation = _impl,", + " attrs = {", + " '_allowlist_test': attr.label(", + " default = '//allowlist:allowlist',", + " cfg = 'exec',", + " providers = [PackageSpecificationInfo]", + " ),", + " },", + ")"); + scratch.file( + "test/BUILD", + "load('//test:rule.bzl', 'custom_rule')", + "custom_rule(name='allowlist_rule')"); + + getConfiguredTarget("//test:allowlist_rule"); + + assertNoEvents(); + } + + @Test + public void targetNotInAllowlist_negativePath() throws Exception { + scratch.file( + "allowlist/BUILD", + "package_group(", + " name='allowlist',", + " packages=[", + " '-//direct',", + " ])"); + scratch.file( + "test/rule.bzl", + "def _impl(ctx):", + " target = '//direct:rule_from_allowlist'", + " target_in_allowlist =" + + " ctx.attr._allowlist_test[PackageSpecificationInfo].contains(target)", + " if target_in_allowlist:", + " fail('Target should not be in the allowlist (negative path)')", + " return []", + "custom_rule = rule(", + " implementation = _impl,", + " attrs = {", + " '_allowlist_test': attr.label(", + " default = '//allowlist:allowlist',", + " cfg = 'exec',", + " providers = [PackageSpecificationInfo]", + " ),", + " },", + ")"); + scratch.file( + "test/BUILD", + "load('//test:rule.bzl', 'custom_rule')", + "custom_rule(name='allowlist_rule')"); + + getConfiguredTarget("//test:allowlist_rule"); + + assertNoEvents(); + } } From 4dcc271070acc9fc66c6a279916077e9193d52f4 Mon Sep 17 00:00:00 2001 From: "bazel.build machine account" <15028808+bazel-io@users.noreply.github.com> Date: Thu, 7 Sep 2023 07:56:29 -0400 Subject: [PATCH 048/125] [6.4.0] Wrong include path to Clang 16 on Windows (#19430) Clang install on Windows started using just the major version number in the install path rather than the full version number. Fixes https://github.com/bazelbuild/bazel/issues/17863 Closes #19391. Commit https://github.com/bazelbuild/bazel/commit/0377badd52612a3c2185d3e392c53a95fd38890a PiperOrigin-RevId: 562480023 Change-Id: Iebd5d3cedff48739747fa8668d56ff8f1d9350b9 Co-authored-by: Orion Hodson --- tools/cpp/windows_cc_configure.bzl | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tools/cpp/windows_cc_configure.bzl b/tools/cpp/windows_cc_configure.bzl index 5f526ecd1d8bc7..760a8435f99a65 100644 --- a/tools/cpp/windows_cc_configure.bzl +++ b/tools/cpp/windows_cc_configure.bzl @@ -565,6 +565,18 @@ def _get_clang_version(repository_ctx, clang_cl): auto_configure_fail("Failed to get clang version by running \"%s -v\"" % clang_cl) return first_line.split(" ")[-1] +def _get_clang_dir(repository_ctx, llvm_path, clang_version): + """Get the clang installation directory.""" + + # The clang_version string format is "X.X.X" + clang_dir = llvm_path + "\\lib\\clang\\" + clang_version + if repository_ctx.path(clang_dir).exists: + return clang_dir + + # Clang 16 changed the install path to use just the major number. + clang_major_version = clang_version.split(".")[0] + return llvm_path + "\\lib\\clang\\" + clang_major_version + def _get_msys_mingw_vars(repository_ctx): """Get the variables we need to populate the msys/mingw toolchains.""" tool_paths, tool_bin_path, inc_dir_msys = _get_escaped_windows_msys_starlark_content(repository_ctx) @@ -658,7 +670,7 @@ def _get_msvc_vars(repository_ctx, paths, target_arch = "x64", msvc_vars_x64 = N escaped_cxx_include_directories.append("\"%s\"" % path) if llvm_path: clang_version = _get_clang_version(repository_ctx, build_tools["CL"]) - clang_dir = llvm_path + "\\lib\\clang\\" + clang_version + clang_dir = _get_clang_dir(repository_ctx, llvm_path, clang_version) clang_include_path = (clang_dir + "\\include").replace("\\", "\\\\") escaped_cxx_include_directories.append("\"%s\"" % clang_include_path) clang_lib_path = (clang_dir + "\\lib\\windows").replace("\\", "\\\\") @@ -734,7 +746,7 @@ def _get_clang_cl_vars(repository_ctx, paths, msvc_vars, target_arch): llvm_lib_path = find_llvm_tool(repository_ctx, llvm_path, "llvm-lib.exe") clang_version = _get_clang_version(repository_ctx, clang_cl_path) - clang_dir = llvm_path + "\\lib\\clang\\" + clang_version + clang_dir = _get_clang_dir(repository_ctx, llvm_path, clang_version) clang_include_path = (clang_dir + "\\include").replace("\\", "\\\\") clang_lib_path = (clang_dir + "\\lib\\windows").replace("\\", "\\\\") From 3aea7d747e1ca436fe1b757e97c39d6455bf97e1 Mon Sep 17 00:00:00 2001 From: "Ian (Hee) Cha" Date: Thu, 7 Sep 2023 15:00:24 -0700 Subject: [PATCH 049/125] =?UTF-8?q?[6.4.0]=20Simplify=20release=20notes=20?= =?UTF-8?q?by=20just=20printing=20the=20first=20line=20of=20the=20commit?= =?UTF-8?q?=20=E2=80=A6=20(#19448)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …(usually the PR title) for minor and patch releases instead of looking for the "relnotes:" marker Commit https://github.com/bazelbuild/bazel/commit/d2e8b7e6d809799e8d2660b888383e00867972b1 PiperOrigin-RevId: 555265560 Change-Id: I5fae2ce763bef6d9b989212a8f3aba66777fb561 Co-authored-by: Googler --- scripts/release/relnotes.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/scripts/release/relnotes.py b/scripts/release/relnotes.py index da3c9fe1ad0678..a90c4335855083 100644 --- a/scripts/release/relnotes.py +++ b/scripts/release/relnotes.py @@ -26,7 +26,14 @@ def git(*args): list(args)).decode("utf-8").strip().split("\n") -def extract_relnotes(commit_message_lines, is_major_release): +def extract_pr_title(commit_message_lines): + """Extracts first line from commit message (passed in as a list of lines).""" + return re.sub( + r"\[\d+\.\d+\.\d\]\s?", "", commit_message_lines[0].strip() + ) + + +def extract_relnotes(commit_message_lines): """Extracts relnotes from a commit message (passed in as a list of lines).""" relnote_lines = [] in_relnote = False @@ -45,18 +52,7 @@ def extract_relnotes(commit_message_lines, is_major_release): relnote = " ".join(relnote_lines) relnote_lower = relnote.strip().lower().rstrip(".") if relnote_lower == "n/a" or relnote_lower == "none" or not relnote_lower: - if is_major_release: - return None - relnote = re.sub( - r"\[\d+\.\d+\.\d\]\s?", "", commit_message_lines[0].strip() - ) - else: - issue_id = re.search( - r"\(\#[0-9]+\)$", commit_message_lines[0].strip().split()[-1] - ) - if issue_id: - relnote = relnote + " " + issue_id.group(0).strip() - + return None return relnote @@ -78,7 +74,9 @@ def get_relnotes_between(base, head, is_major_release): rolled_back_commits.add(m[1]) # The rollback commit itself is also skipped. continue - relnote = extract_relnotes(lines, is_major_release) + relnote = ( + extract_relnotes(lines) if is_major_release else extract_pr_title(lines) + ) if relnote is not None: relnotes.append(relnote) return relnotes From 2b93256897b3845649872a18bfec606f3afa7d35 Mon Sep 17 00:00:00 2001 From: kotlaja Date: Fri, 8 Sep 2023 11:17:37 +0200 Subject: [PATCH 050/125] [6.4.0] Remove PackageGroupConfiguredTarget.isAvailableFor function (#19444) Support for checking if a target exists inside an allowlist was added inside PackageSpecificationProvider as a `contains` function (c4d5fdb). `PackageGroupConfiguredTarget.isAvailableFor` can be removed. Cherrypicking beb6ea8. --- .../PackageGroupConfiguredTarget.java | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/analysis/configuredtargets/PackageGroupConfiguredTarget.java b/src/main/java/com/google/devtools/build/lib/analysis/configuredtargets/PackageGroupConfiguredTarget.java index f2327eae22b61a..56291e51dabaa5 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/configuredtargets/PackageGroupConfiguredTarget.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/configuredtargets/PackageGroupConfiguredTarget.java @@ -14,15 +14,11 @@ package com.google.devtools.build.lib.analysis.configuredtargets; -import static net.starlark.java.eval.Module.ofInnermostEnclosingStarlarkFunction; - import com.google.devtools.build.lib.actions.Artifact; -import com.google.devtools.build.lib.analysis.Allowlist; import com.google.devtools.build.lib.analysis.FileProvider; import com.google.devtools.build.lib.analysis.PackageSpecificationProvider; import com.google.devtools.build.lib.analysis.TargetContext; import com.google.devtools.build.lib.analysis.TransitiveInfoProvider; -import com.google.devtools.build.lib.cmdline.BazelModuleContext; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.collect.nestedset.NestedSet; @@ -32,12 +28,6 @@ import com.google.devtools.build.lib.packages.PackageSpecification.PackageGroupContents; import com.google.devtools.build.lib.packages.Provider; import javax.annotation.Nullable; -import net.starlark.java.annot.Param; -import net.starlark.java.annot.ParamType; -import net.starlark.java.annot.StarlarkMethod; -import net.starlark.java.eval.EvalException; -import net.starlark.java.eval.Starlark; -import net.starlark.java.eval.StarlarkThread; /** * Dummy ConfiguredTarget for package groups. Contains no functionality, since package groups are @@ -86,24 +76,4 @@ protected Info rawGetStarlarkProvider(Provider.Key providerKey) { protected Object rawGetStarlarkProvider(String providerKey) { return null; } - - @StarlarkMethod( - name = "isAvailableFor", - documented = false, - parameters = { - @Param( - name = "label", - allowedTypes = {@ParamType(type = Label.class)}) - }, - useStarlarkThread = true) - public boolean starlarkMatches(Label label, StarlarkThread starlarkThread) throws EvalException { - RepositoryName repository = - BazelModuleContext.of(ofInnermostEnclosingStarlarkFunction(starlarkThread)) - .label() - .getRepository(); - if (!"@_builtins".equals(repository.getNameWithAt())) { - throw Starlark.errorf("private API only for use by builtins"); - } - return Allowlist.isAvailableFor(packageSpecificationProvider.getPackageSpecifications(), label); - } } From ba8c5fe7019209c8241a90d20add544efc0b02b5 Mon Sep 17 00:00:00 2001 From: "bazel.build machine account" <15028808+bazel-io@users.noreply.github.com> Date: Fri, 8 Sep 2023 14:21:46 -0400 Subject: [PATCH 051/125] [6.4.0] Remove default -s flag from macOS libtool invocation (#19454) -s is the default but it's not supported by llvm's libtool replacement which results in failed links. I don't think we really care about the specifics here, so since it's the default for most folks we can drop it. Closes #17489. Commit https://github.com/bazelbuild/bazel/commit/ae7cfa59461b2c694226be689662d387e9c38427 PiperOrigin-RevId: 511121405 Change-Id: Id0b52f960072100d232fdbf5461dec068c8e3edf Co-authored-by: Keith Smiley --- .../google/devtools/build/lib/rules/cpp/CppActionConfigs.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionConfigs.java b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionConfigs.java index 1ade3e2dfc0103..e7561c2639745c 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionConfigs.java +++ b/src/main/java/com/google/devtools/build/lib/rules/cpp/CppActionConfigs.java @@ -631,7 +631,7 @@ public static ImmutableList getLegacyFeatures( " action: 'c++-link-static-library'", " flag_group {", ifLinux(platform, "flag: 'rcsD'"), - ifMac(platform, "flag: '-static'", "flag: '-s'"), + ifMac(platform, "flag: '-static'"), " }", " flag_group {", " expand_if_all_available: 'output_execpath'", From 900ab0c8cda3a560b04f1e737f4a8d042dbff5f2 Mon Sep 17 00:00:00 2001 From: "Ian (Hee) Cha" Date: Fri, 8 Sep 2023 12:19:38 -0700 Subject: [PATCH 052/125] [6.4.0] Turn off lockfile feature by default (#19462) Turn on the lockfile feature by default in bazel 7. But until then, we are turning it off. --- .../devtools/build/lib/bazel/repository/RepositoryOptions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java index d11631b4556502..1285c1a2807f5c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java @@ -278,7 +278,7 @@ public class RepositoryOptions extends OptionsBase { @Option( name = "lockfile_mode", converter = LockfileMode.Converter.class, - defaultValue = "update", + defaultValue = "off", documentationCategory = OptionDocumentationCategory.BZLMOD, effectTags = {OptionEffectTag.LOADING_AND_ANALYSIS}, help = From cb187a08e6735a42be9aecf53a143144e4cc9a49 Mon Sep 17 00:00:00 2001 From: Tiago Quelhas Date: Fri, 8 Sep 2023 23:07:17 +0200 Subject: [PATCH 053/125] [6.4.0] Take the no-remote-exec tag into account when computing the action salt (#19457) Multiple times have I noticed build breakages, where people add/remove the "no-remote-exec" annotation from targets, which only came to light after existing remote cache entries expire. For example, people may try to promote a test to run remotely. "bazel test" shows the test passes, so they assume the test is safe to run remotely. Later on, the cache entry for that test expires. Or the test changes, causing its action cache key to be different. The test gets rebuilt remotely. This now fails, because the test never succeeded remotely to begin with. The cache entry belonged to the action that ran locally. Let's formalize the schema of the "salt" field. Instead of reusing the Platform message, let's declare our own message that we can use to add arbitrary properties that should be taken into account when determining the action's uniqueness. Closes #17304. PiperOrigin-RevId: 505010966 Change-Id: Ib88e769795f18b1724895ebd79835b2bc3b13e1e Co-authored-by: Ed Schouten --- .../google/devtools/build/lib/remote/BUILD | 1 + .../lib/remote/RemoteExecutionService.java | 13 +++++------ src/main/protobuf/spawn.proto | 14 ++++++++++++ .../google/devtools/build/lib/remote/BUILD | 1 + .../remote/RemoteExecutionServiceTest.java | 22 +++++++++++++++---- .../lib/remote/RemoteSpawnRunnerTest.java | 4 ++-- 6 files changed, 42 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/BUILD b/src/main/java/com/google/devtools/build/lib/remote/BUILD index 4dd5629f1b1a49..04a36e0b999904 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/BUILD +++ b/src/main/java/com/google/devtools/build/lib/remote/BUILD @@ -117,6 +117,7 @@ java_library( "//src/main/java/com/google/devtools/build/skyframe", "//src/main/java/com/google/devtools/common/options", "//src/main/protobuf:failure_details_java_proto", + "//src/main/protobuf:spawn_java_proto", "//third_party:auth", "//third_party:caffeine", "//third_party:flogger", diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java index c3545b669c338a..ecf8f50166b880 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java @@ -77,6 +77,7 @@ import com.google.devtools.build.lib.buildtool.buildevent.BuildInterruptedEvent; import com.google.devtools.build.lib.events.Event; import com.google.devtools.build.lib.events.Reporter; +import com.google.devtools.build.lib.exec.Protos.CacheSalt; import com.google.devtools.build.lib.exec.SpawnInputExpander.InputWalker; import com.google.devtools.build.lib.exec.SpawnRunner.SpawnExecutionContext; import com.google.devtools.build.lib.exec.local.LocalEnvProvider; @@ -475,18 +476,16 @@ public MerkleTree uncachedBuildMerkleTreeVisitor( @Nullable private static ByteString buildSalt(Spawn spawn) { + CacheSalt.Builder saltBuilder = + CacheSalt.newBuilder().setMayBeExecutedRemotely(Spawns.mayBeExecutedRemotely(spawn)); + String workspace = spawn.getExecutionInfo().get(ExecutionRequirements.DIFFERENTIATE_WORKSPACE_CACHE); if (workspace != null) { - Platform platform = - Platform.newBuilder() - .addProperties( - Platform.Property.newBuilder().setName("workspace").setValue(workspace).build()) - .build(); - return platform.toByteString(); + saltBuilder.setWorkspace(workspace); } - return null; + return saltBuilder.build().toByteString(); } /** diff --git a/src/main/protobuf/spawn.proto b/src/main/protobuf/spawn.proto index f11de34f13d177..350ff407008a8c 100644 --- a/src/main/protobuf/spawn.proto +++ b/src/main/protobuf/spawn.proto @@ -185,3 +185,17 @@ message SpawnExec { // Timing, size and memory statistics. SpawnMetrics metrics = 20; } + +// Additional information that should be taken into account when +// computing the key of an action, thereby ensuring that actions remain +// distinct. +message CacheSalt { + // Whether or not the action may be executed remotely, if remote + // execution were to be enabled. This ensures that adding/removing the + // "no-remote-exec" tag from a target forces a local/remote rebuild. + bool may_be_executed_remotely = 1; + + // Requires the execution service do NOT share caches across different + // workspace. + string workspace = 2; +} diff --git a/src/test/java/com/google/devtools/build/lib/remote/BUILD b/src/test/java/com/google/devtools/build/lib/remote/BUILD index 7c83743208ff34..bcdc577ff5ef0e 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/BUILD +++ b/src/test/java/com/google/devtools/build/lib/remote/BUILD @@ -98,6 +98,7 @@ java_test( "//src/main/java/com/google/devtools/common/options", "//src/main/protobuf:failure_details_java_proto", "//src/main/protobuf:remote_execution_log_java_proto", + "//src/main/protobuf:spawn_java_proto", "//src/test/java/com/google/devtools/build/lib:test_runner", "//src/test/java/com/google/devtools/build/lib/actions/util", "//src/test/java/com/google/devtools/build/lib/analysis/util", diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java index 8b17b1433feaff..fcc728e7cf7814 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteExecutionServiceTest.java @@ -80,6 +80,7 @@ import com.google.devtools.build.lib.events.EventKind; import com.google.devtools.build.lib.events.Reporter; import com.google.devtools.build.lib.events.StoredEventHandler; +import com.google.devtools.build.lib.exec.Protos.CacheSalt; import com.google.devtools.build.lib.exec.util.FakeOwner; import com.google.devtools.build.lib.exec.util.SpawnBuilder; import com.google.devtools.build.lib.remote.RemoteExecutionService.RemoteActionResult; @@ -261,7 +262,7 @@ public void buildRemoteAction_withActionInputDirectoryAsOutput() throws Exceptio } @Test - public void buildRemoteAction_differentiateWorkspace_generateActionSalt() throws Exception { + public void buildRemoteAction_generateActionSalt_differentiateWorkspaceCache() throws Exception { Spawn spawn = new SpawnBuilder("dummy") .withExecutionInfo(ExecutionRequirements.DIFFERENTIATE_WORKSPACE_CACHE, "aa") @@ -271,10 +272,23 @@ public void buildRemoteAction_differentiateWorkspace_generateActionSalt() throws RemoteAction remoteAction = service.buildRemoteAction(spawn, context); - Platform expected = - Platform.newBuilder() - .addProperties(Platform.Property.newBuilder().setName("workspace").setValue("aa")) + CacheSalt expected = + CacheSalt.newBuilder().setMayBeExecutedRemotely(true).setWorkspace("aa").build(); + assertThat(remoteAction.getAction().getSalt()).isEqualTo(expected.toByteString()); + } + + @Test + public void buildRemoteAction_generateActionSalt_noRemoteExec() throws Exception { + Spawn spawn = + new SpawnBuilder("dummy") + .withExecutionInfo(ExecutionRequirements.NO_REMOTE_EXEC, "") .build(); + FakeSpawnExecutionContext context = newSpawnExecutionContext(spawn); + RemoteExecutionService service = newRemoteExecutionService(); + + RemoteAction remoteAction = service.buildRemoteAction(spawn, context); + + CacheSalt expected = CacheSalt.newBuilder().setMayBeExecutedRemotely(false).build(); assertThat(remoteAction.getAction().getSalt()).isEqualTo(expected.toByteString()); } diff --git a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java index 623f2fe0808fb1..b479751e86fab3 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/RemoteSpawnRunnerTest.java @@ -152,7 +152,7 @@ public class RemoteSpawnRunnerTest { // The action key of the Spawn returned by newSimpleSpawn(). private final String simpleActionId = - "eb45b20cc979d504f96b9efc9a08c48103c6f017afa09c0df5c70a5f92a98ea8"; + "31aea267dc597b047a9b6993100415b6406f82822318dc8988e4164a535b51ee"; @Before public final void setUp() throws Exception { @@ -557,7 +557,7 @@ public void testHumanReadableServerLogsSavedForFailingActionWithSiblingRepositor Digest logDigest = digestUtil.computeAsUtf8("bla"); Path logPath = logDir - .getRelative("b9a727771337fd8ce54821f4805e2d451c4739e92fec6f8ecdb18ff9d1983b27") + .getRelative("e0a5a3561464123504c1240b3587779cdfd6adee20f72aa136e388ecfd570c12") .getRelative("logname"); ExecuteResponse resp = ExecuteResponse.newBuilder() From 6052b8bbbf74e1b2c3112fe74663a5d1b5f6b9e8 Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Fri, 8 Sep 2023 23:40:42 +0200 Subject: [PATCH 054/125] [6.4.0] Add `--incompatible_merge_fixed_and_default_shell_env` (#19319) With the new flag, Starlark actions that specify both `env` and `use_default_shell_env` will no longer have `env` ignored. Instead, the values of `env` will override the default shell environment. This allows Starlark actions to both pick up user-configured variables such as `PATH` from the shell environment as well as set variables to fixed values required for the action, e.g., variables provided by the C++ toolchain. Rationale for having `env` override the default shell env: The rules know best which values they have to set specific environment variables to in order to successfully execute an action, so a situation where users break an action by a globally applied `--action_env` is prevented. If users really do have to be able to modify an environment variables fixed by the rule, the rule can always make this configurable via an attribute. Work towards #5980 Fixes #12049 Closes #18235. Commit https://github.com/bazelbuild/bazel/commit/d1fdc5303fd3cc22c5091aa4ce7df02eef09d922 PiperOrigin-RevId: 559506535 Change-Id: I7ec6ae17b076bbca72fab8394f3a8b3e4f9ea9d8 Fixes #19312 --------- Co-authored-by: Ivo List Co-authored-by: Ian (Hee) Cha --- .../lib/analysis/actions/SpawnAction.java | 53 +++++++++++----- .../starlark/StarlarkActionFactory.java | 19 ++++-- .../semantics/BuildLanguageOptions.java | 18 ++++++ src/test/shell/integration/action_env_test.sh | 61 ++++++++++++++++++- 4 files changed, 130 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java b/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java index 43d34f565f99ff..c2c1cfe9b46736 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/actions/SpawnAction.java @@ -28,6 +28,7 @@ import com.google.common.collect.Interner; import com.google.common.collect.Iterables; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.collect.Sets; import com.google.devtools.build.lib.actions.AbstractAction; import com.google.devtools.build.lib.actions.ActionContinuationOrResult; import com.google.devtools.build.lib.actions.ActionEnvironment; @@ -683,7 +684,6 @@ public static class Builder { private final List outputs = new ArrayList<>(); private final List inputRunfilesSuppliers = new ArrayList<>(); private ResourceSetOrBuilder resourceSetOrBuilder = AbstractAction.DEFAULT_RESOURCE_SET; - private ActionEnvironment actionEnvironment = null; private ImmutableMap environment = ImmutableMap.of(); private ImmutableSet inheritedEnvironment = ImmutableSet.of(); private ImmutableMap executionInfo = ImmutableMap.of(); @@ -717,7 +717,6 @@ public Builder(Builder other) { this.outputs.addAll(other.outputs); this.inputRunfilesSuppliers.addAll(other.inputRunfilesSuppliers); this.resourceSetOrBuilder = other.resourceSetOrBuilder; - this.actionEnvironment = other.actionEnvironment; this.environment = other.environment; this.executionInfo = other.executionInfo; this.isShellCommand = other.isShellCommand; @@ -762,12 +761,31 @@ public SpawnAction build(ActionOwner owner, BuildConfigurationValue configuratio result.addCommandLine(pair); } CommandLines commandLines = result.build(); - ActionEnvironment env = - actionEnvironment != null - ? actionEnvironment - : useDefaultShellEnvironment - ? configuration.getActionEnvironment() - : ActionEnvironment.create(environment, inheritedEnvironment); + ActionEnvironment env; + if (useDefaultShellEnvironment && environment != null) { + // Inherited variables override fixed variables in ActionEnvironment. Since we want the + // fixed part of the action-provided environment to override the inherited part of the + // user-provided environment, we have to explicitly filter the inherited part. + var userFilteredInheritedEnv = + ImmutableSet.copyOf( + Sets.difference( + configuration.getActionEnvironment().getInheritedEnv(), environment.keySet())); + // Do not create a new ActionEnvironment in the common case where no vars have been filtered + // out. + if (userFilteredInheritedEnv.equals( + configuration.getActionEnvironment().getInheritedEnv())) { + env = configuration.getActionEnvironment(); + } else { + env = + ActionEnvironment.create( + configuration.getActionEnvironment().getFixedEnv(), userFilteredInheritedEnv); + } + env = env.withAdditionalFixedVariables(environment); + } else if (useDefaultShellEnvironment) { + env = configuration.getActionEnvironment(); + } else { + env = ActionEnvironment.create(environment, inheritedEnvironment); + } return buildSpawnAction( owner, commandLines, configuration.getCommandLineLimits(), configuration, env); } @@ -984,13 +1002,6 @@ public Builder setResources(ResourceSetOrBuilder resourceSetOrBuilder) { return this; } - /** Sets the action environment. */ - @CanIgnoreReturnValue - public Builder setEnvironment(ActionEnvironment actionEnvironment) { - this.actionEnvironment = actionEnvironment; - return this; - } - /** * Sets the map of environment variables. Do not use! This makes the builder ignore the 'default * shell environment', which is computed from the --action_env command line option. @@ -1075,6 +1086,18 @@ public Builder executeUnconditionally() { return this; } + /** + * Same as {@link #useDefaultShellEnvironment()}, but additionally sets the provided fixed + * environment variables, which take precedence over the variables contained in the default + * shell environment. + */ + @CanIgnoreReturnValue + public Builder useDefaultShellEnvironmentWithOverrides(Map environment) { + this.environment = ImmutableMap.copyOf(environment); + this.useDefaultShellEnvironment = true; + return this; + } + /** * Sets the executable path; the path is interpreted relative to the execution root, unless it's * a bare file name. diff --git a/src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkActionFactory.java b/src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkActionFactory.java index ca8d46a9ce1671..b6058da30c971b 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkActionFactory.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/starlark/StarlarkActionFactory.java @@ -671,15 +671,24 @@ private void registerStarlarkAction( } catch (IllegalArgumentException e) { throw Starlark.errorf("%s", e.getMessage()); } - if (envUnchecked != Starlark.NONE) { - builder.setEnvironment( - ImmutableMap.copyOf(Dict.cast(envUnchecked, String.class, String.class, "env"))); - } if (progressMessage != Starlark.NONE) { builder.setProgressMessageFromStarlark((String) progressMessage); } + + ImmutableMap env = null; + if (envUnchecked != Starlark.NONE) { + env = ImmutableMap.copyOf(Dict.cast(envUnchecked, String.class, String.class, "env")); + } if (Starlark.truth(useDefaultShellEnv)) { - builder.useDefaultShellEnvironment(); + if (env != null + && getSemantics() + .getBool(BuildLanguageOptions.INCOMPATIBLE_MERGE_FIXED_AND_DEFAULT_SHELL_ENV)) { + builder.useDefaultShellEnvironmentWithOverrides(env); + } else { + builder.useDefaultShellEnvironment(); + } + } else if (env != null) { + builder.setEnvironment(env); } RuleContext ruleContext = getRuleContext(); diff --git a/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java b/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java index 2259ab5a68e7d5..469427b6f32856 100644 --- a/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java +++ b/src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java @@ -662,6 +662,19 @@ public final class BuildLanguageOptions extends OptionsBase { help = "If enabled, targets that have unknown attributes set to None fail.") public boolean incompatibleFailOnUnknownAttributes; + @Option( + name = "incompatible_merge_fixed_and_default_shell_env", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.STARLARK_SEMANTICS, + effectTags = {OptionEffectTag.LOADING_AND_ANALYSIS}, + metadataTags = {OptionMetadataTag.INCOMPATIBLE_CHANGE}, + help = + "If enabled, actions registered with ctx.actions.run and ctx.actions.run_shell with both" + + " 'env' and 'use_default_shell_env = True' specified will use an environment" + + " obtained from the default shell environment by overriding with the values passed" + + " in to 'env'. If disabled, the value of 'env' is completely ignored in this case.") + public boolean incompatibleMergeFixedAndDefaultShellEnv; + @Option( name = "incompatible_disable_objc_library_transition", defaultValue = "false", @@ -765,6 +778,9 @@ public StarlarkSemantics toStarlarkSemantics() { EXPERIMENTAL_GET_FIXED_CONFIGURED_ACTION_ENV, experimentalGetFixedConfiguredEnvironment) .setBool(INCOMPATIBLE_FAIL_ON_UNKNOWN_ATTRIBUTES, incompatibleFailOnUnknownAttributes) + .setBool( + INCOMPATIBLE_MERGE_FIXED_AND_DEFAULT_SHELL_ENV, + incompatibleMergeFixedAndDefaultShellEnv) .setBool( INCOMPATIBLE_DISABLE_OBJC_LIBRARY_TRANSITION, incompatibleDisableObjcLibraryTransition) @@ -858,6 +874,8 @@ public StarlarkSemantics toStarlarkSemantics() { "-experimental_get_fixed_configured_action_env"; public static final String INCOMPATIBLE_FAIL_ON_UNKNOWN_ATTRIBUTES = "-incompatible_fail_on_unknown_attributes"; + public static final String INCOMPATIBLE_MERGE_FIXED_AND_DEFAULT_SHELL_ENV = + "-experimental_merge_fixed_and_default_shell_env"; public static final String INCOMPATIBLE_DISABLE_OBJC_LIBRARY_TRANSITION = "-incompatible_disable_objc_library_transition"; diff --git a/src/test/shell/integration/action_env_test.sh b/src/test/shell/integration/action_env_test.sh index 08af2abd059630..6feaf075740bb0 100755 --- a/src/test/shell/integration/action_env_test.sh +++ b/src/test/shell/integration/action_env_test.sh @@ -39,6 +39,15 @@ load("//pkg:build.bzl", "environ") environ(name = "no_default_env", env = 0) environ(name = "with_default_env", env = 1) +environ( + name = "with_default_and_fixed_env", + env = 1, + fixed_env = { + "ACTION_FIXED": "action", + "ACTION_AND_CLIENT_FIXED": "action", + "ACTION_AND_CLIENT_INHERITED": "action", + }, +) sh_test( name = "test_env_foo", @@ -72,12 +81,16 @@ def _impl(ctx): ctx.actions.run_shell( inputs=[], outputs=[output], + env = ctx.attr.fixed_env, use_default_shell_env = ctx.attr.env, command="env > %s" % output.path) environ = rule( implementation=_impl, - attrs={"env": attr.bool(default=True)}, + attrs={ + "env": attr.bool(default=True), + "fixed_env": attr.string_dict(), + }, outputs={"out": "%{name}.env"}, ) EOF @@ -222,6 +235,52 @@ function test_use_default_shell_env { && fail "dynamic action_env used, even though requested not to") || true } +function test_use_default_shell_env_and_fixed_env { + ACTION_AND_CLIENT_INHERITED=client CLIENT_INHERITED=client \ + bazel build \ + --noincompatible_merge_fixed_and_default_shell_env \ + --action_env=ACTION_AND_CLIENT_FIXED=client \ + --action_env=ACTION_AND_CLIENT_INHERITED \ + --action_env=CLIENT_FIXED=client \ + --action_env=CLIENT_INHERITED \ + //pkg:with_default_and_fixed_env + echo + cat bazel-bin/pkg/with_default_and_fixed_env.env + echo + grep -q ACTION_AND_CLIENT_FIXED=client bazel-bin/pkg/with_default_and_fixed_env.env \ + || fail "static action environment not honored" + grep -q ACTION_AND_CLIENT_INHERITED=client bazel-bin/pkg/with_default_and_fixed_env.env \ + || fail "dynamic action environment not honored" + grep -q ACTION_FIXED bazel-bin/pkg/with_default_and_fixed_env.env \ + && fail "fixed env provided by action should have been ignored" + grep -q CLIENT_FIXED=client bazel-bin/pkg/with_default_and_fixed_env.env \ + || fail "static action environment not honored" + grep -q CLIENT_INHERITED=client bazel-bin/pkg/with_default_and_fixed_env.env \ + || fail "dynamic action environment not honored" + + ACTION_AND_CLIENT_INHERITED=client CLIENT_INHERITED=client \ + bazel build \ + --incompatible_merge_fixed_and_default_shell_env \ + --action_env=ACTION_AND_CLIENT_FIXED=client \ + --action_env=ACTION_AND_CLIENT_INHERITED \ + --action_env=CLIENT_FIXED=client \ + --action_env=CLIENT_INHERITED \ + //pkg:with_default_and_fixed_env + echo + cat bazel-bin/pkg/with_default_and_fixed_env.env + echo + grep -q ACTION_AND_CLIENT_FIXED=action bazel-bin/pkg/with_default_and_fixed_env.env \ + || fail "action-provided env should have overridden static --action_env" + grep -q ACTION_AND_CLIENT_INHERITED=action bazel-bin/pkg/with_default_and_fixed_env.env \ + || fail "action-provided env should have overridden dynamic --action_env" + grep -q ACTION_FIXED=action bazel-bin/pkg/with_default_and_fixed_env.env \ + || fail "action-provided env should have been honored" + grep -q CLIENT_FIXED=client bazel-bin/pkg/with_default_and_fixed_env.env \ + || fail "static action environment not honored" + grep -q CLIENT_INHERITED=client bazel-bin/pkg/with_default_and_fixed_env.env \ + || fail "dynamic action environment not honored" +} + function test_action_env_changes_honored { # Verify that changes to the explicitly specified action_env in honored in # tests. Regression test for #3265. From 5a0339fd9d6123badd5bdac25c2a3c70cefb67b0 Mon Sep 17 00:00:00 2001 From: "bazel.build machine account" <15028808+bazel-io@users.noreply.github.com> Date: Mon, 11 Sep 2023 14:26:17 -0400 Subject: [PATCH 055/125] [6.4.0] Improve error when a label is provided in `config_setting`'s `values` (#19484) Suggest to use `flag_values` instead. Previously, this only resulted in an ambiguous "Unrecognized option" error. Closes #19464. Commit https://github.com/bazelbuild/bazel/commit/bd5dbc5a4fb890758115cce60a8ef8e5b09c6cf1 PiperOrigin-RevId: 564367170 Change-Id: Ibe9397be4300783ab6e5b5de9e08e40142de0b4c Co-authored-by: Fabian Meumertzheim --- .../build/lib/rules/config/ConfigSetting.java | 15 +++++++++++++++ .../lib/rules/config/ConfigSettingTest.java | 16 ++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/main/java/com/google/devtools/build/lib/rules/config/ConfigSetting.java b/src/main/java/com/google/devtools/build/lib/rules/config/ConfigSetting.java index 3c18636b04d271..274adc7ccc549d 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/config/ConfigSetting.java +++ b/src/main/java/com/google/devtools/build/lib/rules/config/ConfigSetting.java @@ -67,6 +67,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import javax.annotation.Nullable; /** @@ -83,6 +84,20 @@ public ConfiguredTarget create(RuleContext ruleContext) throws InterruptedException, ActionConflictException { AttributeMap attributes = NonconfigurableAttributeMapper.of(ruleContext.getRule()); + Optional likelyLabelInvalidSetting = + attributes.get(ConfigSettingRule.SETTINGS_ATTRIBUTE, Type.STRING_DICT).keySet().stream() + .filter(s -> s.startsWith("@") || s.startsWith("//") || s.startsWith(":")) + .findFirst(); + if (likelyLabelInvalidSetting.isPresent()) { + ruleContext.attributeError( + ConfigSettingRule.SETTINGS_ATTRIBUTE, + String.format( + "'%s' is not a valid setting name, but appears to be a label. Did you mean to place" + + " it in %s instead?", + likelyLabelInvalidSetting.get(), ConfigSettingRule.FLAG_SETTINGS_ATTRIBUTE)); + return null; + } + // Get the built-in Blaze flag settings that match this rule. ImmutableMultimap nativeFlagSettings = ImmutableMultimap.builder() diff --git a/src/test/java/com/google/devtools/build/lib/rules/config/ConfigSettingTest.java b/src/test/java/com/google/devtools/build/lib/rules/config/ConfigSettingTest.java index aa425fb7200c54..3a182ff1dbb81e 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/config/ConfigSettingTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/config/ConfigSettingTest.java @@ -2317,4 +2317,20 @@ public void singleValueThatLooksLikeMultiValueIsOkay() throws Exception { assertThat(getConfiguredTarget("//test:fg")).isNotNull(); assertNoEvents(); } + + @Test + public void labelInValuesError() throws Exception { + scratch.file( + "test/BUILD", + "config_setting(", + " name = 'match',", + " values = {'//foo:bar': 'value'},", + ")"); + reporter.removeHandler(failFastHandler); // expect errors + assertThat(getConfiguredTarget("//test:match")).isNull(); + assertContainsEvent( + "in values attribute of config_setting rule //test:match: '//foo:bar' is" + + " not a valid setting name, but appears to be a label. Did you mean to place it in" + + " flag_values instead?"); + } } From 56d0a049d59b0d71ca9edb413fbc1c0b7a540677 Mon Sep 17 00:00:00 2001 From: Tiago Quelhas Date: Mon, 11 Sep 2023 21:05:29 +0200 Subject: [PATCH 056/125] [6.4.0] Mark tool inputs in the execution log. (#19483) This is useful to identify actions that don't separate their tool and non-tool inputs properly. PiperOrigin-RevId: 564351781 Change-Id: If93c28e1e10f4ba0b2dd4b802d29d34d095bde12 --- .../build/lib/exec/SpawnLogContext.java | 14 ++++- src/main/protobuf/spawn.proto | 4 ++ .../lib/exec/AbstractSpawnStrategyTest.java | 59 +++++++++++++++++++ .../build/lib/exec/util/SpawnBuilder.java | 8 +++ 4 files changed, 84 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java b/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java index 88cfe1b0f5e960..ccb470d373d1d0 100644 --- a/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java +++ b/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java @@ -15,10 +15,12 @@ import build.bazel.remote.execution.v2.Platform; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; import com.google.common.flogger.GoogleLogger; import com.google.common.hash.HashCode; import com.google.devtools.build.lib.actions.ActionContext; import com.google.devtools.build.lib.actions.ActionInput; +import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact; import com.google.devtools.build.lib.actions.ExecException; import com.google.devtools.build.lib.actions.FileArtifactValue; import com.google.devtools.build.lib.actions.MetadataProvider; @@ -100,6 +102,8 @@ public void logSpawn( builder.addEnvironmentVariablesBuilder().setName(var).setValue(env.get(var)); } + ImmutableSet toolFiles = spawn.getToolFiles().toSet(); + try { for (Map.Entry e : inputMap.entrySet()) { ActionInput input = e.getValue(); @@ -111,7 +115,15 @@ public void logSpawn( listDirectoryContents(inputPath, builder::addInputs, metadataProvider); } else { Digest digest = computeDigest(input, null, metadataProvider, xattrProvider); - builder.addInputsBuilder().setPath(input.getExecPathString()).setDigest(digest); + boolean isTool = + toolFiles.contains(input) + || (input instanceof TreeFileArtifact + && toolFiles.contains(((TreeFileArtifact) input).getParent())); + builder + .addInputsBuilder() + .setPath(input.getExecPathString()) + .setDigest(digest) + .setIsTool(isTool); } } } catch (IOException e) { diff --git a/src/main/protobuf/spawn.proto b/src/main/protobuf/spawn.proto index 350ff407008a8c..5863f973424431 100644 --- a/src/main/protobuf/spawn.proto +++ b/src/main/protobuf/spawn.proto @@ -41,6 +41,10 @@ message File { // Digest of the file's contents. Digest digest = 2; + + // Whether the file is a tool. + // Only set for inputs, never for outputs. + bool is_tool = 3; } // Contents of command environment. diff --git a/src/test/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategyTest.java b/src/test/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategyTest.java index a1d315267231cc..dd0a4d1954d07c 100644 --- a/src/test/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategyTest.java +++ b/src/test/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategyTest.java @@ -29,9 +29,12 @@ import com.google.common.collect.ImmutableList; import com.google.devtools.build.lib.actions.ActionExecutionContext; import com.google.devtools.build.lib.actions.Artifact; +import com.google.devtools.build.lib.actions.Artifact.SpecialArtifact; +import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact; import com.google.devtools.build.lib.actions.ArtifactRoot; import com.google.devtools.build.lib.actions.FutureSpawn; import com.google.devtools.build.lib.actions.MetadataProvider; +import com.google.devtools.build.lib.actions.ArtifactRoot.RootType; import com.google.devtools.build.lib.actions.Spawn; import com.google.devtools.build.lib.actions.SpawnExecutedEvent; import com.google.devtools.build.lib.actions.SpawnResult; @@ -92,6 +95,7 @@ public TestedSpawnStrategy(Path execRoot, SpawnRunner spawnRunner) { private final Path execRoot = fs.getPath("/execroot"); private Scratch scratch; private ArtifactRoot rootDir; + private ArtifactRoot outputDir; @Mock private SpawnRunner spawnRunner; @Mock private ActionExecutionContext actionExecutionContext; @Mock private MessageOutputStream messageOutput; @@ -103,6 +107,7 @@ public final void setUp() throws Exception { MockitoAnnotations.initMocks(this); scratch = new Scratch(fs); rootDir = ArtifactRoot.asSourceRoot(Root.fromPath(scratch.dir("/execroot"))); + outputDir = ArtifactRoot.asDerivedRoot(scratch.dir("/execroot"), RootType.Output, "out"); eventHandler = new StoredEventHandler(); when(actionExecutionContext.getEventHandler()).thenReturn(eventHandler); when(actionExecutionContext.getClock()).thenReturn(clock); @@ -453,6 +458,60 @@ public void testLogSpawn_defaultPlatform_getsLogged() throws Exception { verify(messageOutput).write(expected); // output will reflect default properties } + @Test + public void testLogSpawn_toolInputs() throws Exception { + Artifact toolFile = ActionsTestUtil.createArtifact(rootDir, "tool.file"); + SpecialArtifact toolDir = + ActionsTestUtil.createTreeArtifactWithGeneratingAction(outputDir, "tool.dir"); + + scratch.file("/execroot/tool.file", "123"); + scratch.file("/execroot/out/tool.dir/tool.file", "456"); + + setUpExecutionContext(/* executionOptions= */ null, /* remoteOptions= */ null); + when(actionExecutionContext.getArtifactExpander()) + .thenReturn( + (artifact, output) -> { + if (artifact.equals(toolDir)) { + output.add(TreeFileArtifact.createTreeOutput(toolDir, "tool.file")); + } + }); + Spawn spawn = + new SpawnBuilder("cmd").withInputs(toolFile, toolDir).withTools(toolFile, toolDir).build(); + assertThrows( + SpawnExecException.class, + () -> new TestedSpawnStrategy(execRoot, spawnRunner).exec(spawn, actionExecutionContext)); + + SpawnExec expected = + defaultSpawnExecBuilder("cmd") + .addInputs( + File.newBuilder() + .setPath("out/tool.dir/tool.file") + .setDigest( + Digest.newBuilder() + .setHash( + "cdfba543ee8ef7fdb3d8b587648cc22dd792bbd6272cc5447307c7c106c2374c") + .setSizeBytes(4) + .setHashFunctionName("SHA-256") + .build()) + .setIsTool(true) + .build()) + .addInputs( + File.newBuilder() + .setPath("tool.file") + .setDigest( + Digest.newBuilder() + .setHash( + "181210f8f9c779c26da1d9b2075bde0127302ee0e3fca38c9a83f5b1dd8e5d3b") + .setSizeBytes(4) + .setHashFunctionName("SHA-256") + .build()) + .setIsTool(true) + .build()) + .build(); + + verify(messageOutput).write(expected); + } + @Test public void testLogSpawn_spawnMetrics() throws Exception { ExecutionOptions executionOptions = Options.getDefaults(ExecutionOptions.class); diff --git a/src/test/java/com/google/devtools/build/lib/exec/util/SpawnBuilder.java b/src/test/java/com/google/devtools/build/lib/exec/util/SpawnBuilder.java index 9564169e77e2a1..5a396b8d055624 100644 --- a/src/test/java/com/google/devtools/build/lib/exec/util/SpawnBuilder.java +++ b/src/test/java/com/google/devtools/build/lib/exec/util/SpawnBuilder.java @@ -235,6 +235,14 @@ public SpawnBuilder withTool(ActionInput tool) { return this; } + @CanIgnoreReturnValue + public SpawnBuilder withTools(ActionInput... tools) { + for (ActionInput tool : tools) { + this.tools.add(tool); + } + return this; + } + @CanIgnoreReturnValue public SpawnBuilder withLocalResources(ResourceSet resourceSet) { this.resourceSet = resourceSet; From fb4cc46da593c009e9e0153ac0c384f3fc3b90df Mon Sep 17 00:00:00 2001 From: Keith Smiley Date: Mon, 11 Sep 2023 13:40:36 -0700 Subject: [PATCH 057/125] [6.4.0] Add visionOS support (#19436) This cherry picks https://github.com/bazelbuild/bazel/pull/18905 and https://github.com/bazelbuild/bazel/pull/19133 Co-authored-by: nglevin <122120823+nglevin@users.noreply.github.com> --- MODULE.bazel | 2 +- distdir_deps.bzl | 9 +++--- src/MODULE.tools | 2 +- .../build/docgen/templates/be/be-nav.vm | 2 +- .../build/docgen/templates/be/be-toc.vm | 2 +- .../LocalConfigPlatformFunction.java | 2 +- .../rules/apple/AppleCommandLineOptions.java | 18 ++++++++++++ .../lib/rules/apple/AppleConfiguration.java | 28 ++++++++++++++++++- .../build/lib/rules/apple/ApplePlatform.java | 11 ++++++++ .../build/lib/rules/apple/AppleToolchain.java | 2 ++ .../build/lib/rules/apple/XcodeConfig.java | 5 ++++ .../lib/rules/apple/XcodeConfigInfo.java | 16 +++++++++++ .../rules/apple/XcodeVersionProperties.java | 23 ++++++++++++++- .../lib/rules/apple/XcodeVersionRule.java | 8 ++++++ .../lib/rules/apple/XcodeVersionRuleData.java | 6 ++++ .../lib/rules/objc/CompilationSupport.java | 3 +- .../rules/objc/MultiArchBinarySupport.java | 16 +++++++++++ .../build/lib/rules/objc/ObjcRuleClasses.java | 4 +-- .../apple/ApplePlatformTypeApi.java | 3 +- .../apple/XcodeConfigInfoApi.java | 12 ++++++++ .../apple/XcodePropertiesApi.java | 10 +++++++ .../builtins_bzl/common/cc/cc_binary.bzl | 4 ++- .../builtins_bzl/common/objc/transitions.bzl | 12 ++++++++ .../packages/util/MockPlatformSupport.java | 4 +++ .../lib/rules/apple/XcodeConfigTest.java | 18 ++++++++---- .../bazel/android/android_integration_test.sh | 2 +- tools/osx/BUILD | 3 ++ tools/osx/xcode_configure.bzl | 3 ++ tools/osx/xcode_version_flag.bzl | 20 +++++++++++++ 29 files changed, 228 insertions(+), 22 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index dd3094f00effb9..f7ad0d88a95817 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -12,7 +12,7 @@ bazel_dep(name = "rules_license", version = "0.0.3") bazel_dep(name = "bazel_skylib", version = "1.2.0") bazel_dep(name = "protobuf", version = "3.19.6", repo_name = "com_google_protobuf") bazel_dep(name = "grpc", version = "1.47.0", repo_name = "com_github_grpc_grpc") -bazel_dep(name = "platforms", version = "0.0.5") +bazel_dep(name = "platforms", version = "0.0.7") bazel_dep(name = "rules_pkg", version = "0.7.0") bazel_dep(name = "stardoc", version = "0.5.0", repo_name = "io_bazel_skydoc") bazel_dep(name = "zstd-jni", version = "1.5.2-3") diff --git a/distdir_deps.bzl b/distdir_deps.bzl index 712d75cffa8acd..4598ea7803e94c 100644 --- a/distdir_deps.bzl +++ b/distdir_deps.bzl @@ -25,16 +25,17 @@ DIST_DEPS = { # ######################################## "platforms": { - "archive": "platforms-0.0.5.tar.gz", - "sha256": "379113459b0feaf6bfbb584a91874c065078aa673222846ac765f86661c27407", + "archive": "platforms-0.0.7.tar.gz", + "sha256": "3a561c99e7bdbe9173aa653fd579fe849f1d8d67395780ab4770b1f381431d51", "urls": [ - "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.5/platforms-0.0.5.tar.gz", - "https://github.com/bazelbuild/platforms/releases/download/0.0.5/platforms-0.0.5.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.7/platforms-0.0.7.tar.gz", + "https://github.com/bazelbuild/platforms/releases/download/0.0.7/platforms-0.0.7.tar.gz", ], "used_in": [ "additional_distfiles", "test_WORKSPACE_files", ], + "package_version": "0.0.7", }, "bazelci_rules": { "archive": "bazelci_rules-1.0.0.tar.gz", diff --git a/src/MODULE.tools b/src/MODULE.tools index b82be48982cb08..65937922ecfa73 100644 --- a/src/MODULE.tools +++ b/src/MODULE.tools @@ -6,7 +6,7 @@ bazel_dep(name = "rules_license", version = "0.0.3") bazel_dep(name = "rules_proto", version = "4.0.0") bazel_dep(name = "rules_python", version = "0.4.0") -bazel_dep(name = "platforms", version = "0.0.4") +bazel_dep(name = "platforms", version = "0.0.7") bazel_dep(name = "protobuf", version = "3.19.6", repo_name = "com_google_protobuf") bazel_dep(name = "zlib", version = "1.2.13") diff --git a/src/main/java/com/google/devtools/build/docgen/templates/be/be-nav.vm b/src/main/java/com/google/devtools/build/docgen/templates/be/be-nav.vm index 38d7385cf630cd..fb745de8d49aab 100644 --- a/src/main/java/com/google/devtools/build/docgen/templates/be/be-nav.vm +++ b/src/main/java/com/google/devtools/build/docgen/templates/be/be-nav.vm @@ -33,7 +33,7 @@ #end

  • AppEngine
    • -
  • Apple (Swift, iOS, macOS, tvOS, watchOS)
    • +
  • Apple (Swift, iOS, macOS, tvOS, visionOS, watchOS)
  • C#
  • D
  • Docker
    • diff --git a/src/main/java/com/google/devtools/build/docgen/templates/be/be-toc.vm b/src/main/java/com/google/devtools/build/docgen/templates/be/be-toc.vm index bb0d01f416fbea..3821dc6d00fbc6 100644 --- a/src/main/java/com/google/devtools/build/docgen/templates/be/be-toc.vm +++ b/src/main/java/com/google/devtools/build/docgen/templates/be/be-toc.vm @@ -20,7 +20,7 @@ toc: - title: AppEngine path: https://github.com/bazelbuild/rules_appengine status: external - - title: Apple (Swift, iOS, macOS, tvOS, watchOS) + - title: Apple (Swift, iOS, macOS, tvOS, visionOS, watchOS) path: https://github.com/bazelbuild/rules_apple status: external - title: C# diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/LocalConfigPlatformFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/LocalConfigPlatformFunction.java index eecf41a30ecd59..04907ba1d2da6c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/LocalConfigPlatformFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/LocalConfigPlatformFunction.java @@ -162,7 +162,7 @@ private static String moduleFileContent(String repositoryName) { "module(name = \"%s\")", // Try to keep this updated with the src/MODULE.tools file. (Due to MVS, even if this is // not kept up to date, we'll use the latest version anyhow) - "bazel_dep(name = \"platforms\", version = \"0.0.4\")"), + "bazel_dep(name = \"platforms\", version = \"0.0.7\")"), repositoryName); } diff --git a/src/main/java/com/google/devtools/build/lib/rules/apple/AppleCommandLineOptions.java b/src/main/java/com/google/devtools/build/lib/rules/apple/AppleCommandLineOptions.java index 794df2293f98a1..55dc7595c95a6c 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/apple/AppleCommandLineOptions.java +++ b/src/main/java/com/google/devtools/build/lib/rules/apple/AppleCommandLineOptions.java @@ -192,6 +192,10 @@ public class AppleCommandLineOptions extends FragmentOptions { @VisibleForTesting public static final String DEFAULT_TVOS_SDK_VERSION = "9.0"; @VisibleForTesting static final String DEFAULT_IOS_CPU = "x86_64"; + /** The default visionOS CPU value. */ + public static final String DEFAULT_VISIONOS_CPU = + CPU.getCurrent() == CPU.AARCH64 ? "sim_arm64" : "x86_64"; + /** The default watchos CPU value. */ public static final String DEFAULT_WATCHOS_CPU = CPU.getCurrent() == CPU.AARCH64 ? "arm64" : "i386"; @@ -297,6 +301,16 @@ public class AppleCommandLineOptions extends FragmentOptions { + "is a universal binary containing all specified architectures.") public List iosMultiCpus; + @Option( + name = "visionos_cpus", + allowMultiple = true, + converter = CommaSeparatedOptionListConverter.class, + defaultValue = "null", + documentationCategory = OptionDocumentationCategory.OUTPUT_PARAMETERS, + effectTags = {OptionEffectTag.LOSES_INCREMENTAL_STATE, OptionEffectTag.LOADING_AND_ANALYSIS}, + help = "Comma-separated list of architectures for which to build Apple visionOS binaries.") + public List visionosCpus; + @Option( name = "watchos_cpus", allowMultiple = true, @@ -420,6 +434,10 @@ public DottedVersion getMinimumOsVersion() { case TVOS: option = tvosMinimumOs; break; + case VISIONOS: + // TODO: Replace with CppOptions.minimumOsVersion + option = DottedVersion.option(DottedVersion.fromStringUnchecked("1.0")); + break; case WATCHOS: option = watchosMinimumOs; break; diff --git a/src/main/java/com/google/devtools/build/lib/rules/apple/AppleConfiguration.java b/src/main/java/com/google/devtools/build/lib/rules/apple/AppleConfiguration.java index dc7abdb202d7b8..beb2d9f2c000aa 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/apple/AppleConfiguration.java +++ b/src/main/java/com/google/devtools/build/lib/rules/apple/AppleConfiguration.java @@ -115,6 +115,10 @@ public static AppleCpus create(AppleCommandLineOptions options, CoreOptions core (options.iosMultiCpus == null || options.iosMultiCpus.isEmpty()) ? ImmutableList.of(iosCpuFromCpu(coreOptions.cpu)) : ImmutableList.copyOf(options.iosMultiCpus); + ImmutableList visionosCpus = + (options.visionosCpus == null || options.visionosCpus.isEmpty()) + ? ImmutableList.of(AppleCommandLineOptions.DEFAULT_VISIONOS_CPU) + : ImmutableList.copyOf(options.visionosCpus); ImmutableList watchosCpus = (options.watchosCpus == null || options.watchosCpus.isEmpty()) ? ImmutableList.of(AppleCommandLineOptions.DEFAULT_WATCHOS_CPU) @@ -133,13 +137,21 @@ public static AppleCpus create(AppleCommandLineOptions options, CoreOptions core : ImmutableList.copyOf(options.catalystCpus); return new AutoValue_AppleConfiguration_AppleCpus( - appleSplitCpu, iosMultiCpus, watchosCpus, tvosCpus, macosCpus, catalystCpus); + appleSplitCpu, + iosMultiCpus, + visionosCpus, + watchosCpus, + tvosCpus, + macosCpus, + catalystCpus); } abstract String appleSplitCpu(); abstract ImmutableList iosMultiCpus(); + abstract ImmutableList visionosCpus(); + abstract ImmutableList watchosCpus(); abstract ImmutableList tvosCpus(); @@ -248,6 +260,8 @@ private static String getPrefixedAppleCpu(PlatformType applePlatformType, AppleC switch (applePlatformType) { case IOS: return appleCpus.iosMultiCpus().get(0); + case VISIONOS: + return appleCpus.visionosCpus().get(0); case WATCHOS: return appleCpus.watchosCpus().get(0); case TVOS: @@ -296,6 +310,8 @@ public List getMultiArchitectures(PlatformType platformType) { switch (platformType) { case IOS: return appleCpus.iosMultiCpus(); + case VISIONOS: + return appleCpus.visionosCpus(); case WATCHOS: return appleCpus.watchosCpus(); case TVOS: @@ -341,6 +357,14 @@ public ApplePlatform getMultiArchPlatform(PlatformType platformType) { } } return ApplePlatform.IOS_SIMULATOR; + case VISIONOS: + for (String arch : architectures) { + if (ApplePlatform.forTarget(PlatformType.VISIONOS, arch) + == ApplePlatform.VISIONOS_DEVICE) { + return ApplePlatform.VISIONOS_DEVICE; + } + } + return ApplePlatform.VISIONOS_SIMULATOR; case WATCHOS: for (String arch : architectures) { if (ApplePlatform.forTarget(PlatformType.WATCHOS, arch) == ApplePlatform.WATCHOS_DEVICE) { @@ -500,6 +524,8 @@ public enum ConfigurationDistinguisher implements StarlarkValue { UNKNOWN("unknown"), /** Distinguisher for {@code apple_binary} rule with "ios" platform_type. */ APPLEBIN_IOS("applebin_ios"), + /** Distinguisher for {@code apple_binary} rule with "visionos" platform_type. */ + APPLEBIN_VISIONOS("applebin_visionos"), /** Distinguisher for {@code apple_binary} rule with "watchos" platform_type. */ APPLEBIN_WATCHOS("applebin_watchos"), /** Distinguisher for {@code apple_binary} rule with "tvos" platform_type. */ diff --git a/src/main/java/com/google/devtools/build/lib/rules/apple/ApplePlatform.java b/src/main/java/com/google/devtools/build/lib/rules/apple/ApplePlatform.java index a050bbd4fcfd5f..b7af54fb6a9923 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/apple/ApplePlatform.java +++ b/src/main/java/com/google/devtools/build/lib/rules/apple/ApplePlatform.java @@ -37,6 +37,8 @@ public enum ApplePlatform implements ApplePlatformApi { MACOS("macos", "MacOSX", PlatformType.MACOS, true), TVOS_DEVICE("tvos_device", "AppleTVOS", PlatformType.TVOS, true), TVOS_SIMULATOR("tvos_simulator", "AppleTVSimulator", PlatformType.TVOS, false), + VISIONOS_DEVICE("visionos_device", "XROS", PlatformType.VISIONOS, true), + VISIONOS_SIMULATOR("visionos_simulator", "XRSimulator", PlatformType.VISIONOS, false), WATCHOS_DEVICE("watchos_device", "WatchOS", PlatformType.WATCHOS, true), WATCHOS_SIMULATOR("watchos_simulator", "WatchSimulator", PlatformType.WATCHOS, false), CATALYST("catalyst", "MacOSX", PlatformType.CATALYST, true); @@ -45,6 +47,10 @@ public enum ApplePlatform implements ApplePlatformApi { ImmutableSet.of("ios_x86_64", "ios_i386", "ios_sim_arm64"); private static final ImmutableSet IOS_DEVICE_TARGET_CPUS = ImmutableSet.of("ios_armv6", "ios_arm64", "ios_armv7", "ios_armv7s", "ios_arm64e"); + private static final ImmutableSet VISIONOS_SIMULATOR_TARGET_CPUS = + ImmutableSet.of("visionos_x86_64", "visionos_sim_arm64"); + private static final ImmutableSet VISIONOS_DEVICE_TARGET_CPUS = + ImmutableSet.of("visionos_arm64"); private static final ImmutableSet WATCHOS_SIMULATOR_TARGET_CPUS = ImmutableSet.of("watchos_i386", "watchos_x86_64", "watchos_arm64"); private static final ImmutableSet WATCHOS_DEVICE_TARGET_CPUS = @@ -138,6 +144,10 @@ private static ApplePlatform forTargetCpuNullable(String targetCpu) { return IOS_SIMULATOR; } else if (IOS_DEVICE_TARGET_CPUS.contains(targetCpu)) { return IOS_DEVICE; + } else if (VISIONOS_SIMULATOR_TARGET_CPUS.contains(targetCpu)) { + return VISIONOS_SIMULATOR; + } else if (VISIONOS_DEVICE_TARGET_CPUS.contains(targetCpu)) { + return VISIONOS_DEVICE; } else if (WATCHOS_SIMULATOR_TARGET_CPUS.contains(targetCpu)) { return WATCHOS_SIMULATOR; } else if (WATCHOS_DEVICE_TARGET_CPUS.contains(targetCpu)) { @@ -245,6 +255,7 @@ public UnsupportedPlatformTypeException(String msg) { @Immutable public enum PlatformType implements ApplePlatformTypeApi { IOS("ios"), + VISIONOS("visionos"), WATCHOS("watchos"), TVOS("tvos"), MACOS("macos"), diff --git a/src/main/java/com/google/devtools/build/lib/rules/apple/AppleToolchain.java b/src/main/java/com/google/devtools/build/lib/rules/apple/AppleToolchain.java index d042ab80462c72..241a7ecdce91bd 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/apple/AppleToolchain.java +++ b/src/main/java/com/google/devtools/build/lib/rules/apple/AppleToolchain.java @@ -114,6 +114,8 @@ public static String sdkFrameworkDir(ApplePlatform targetPlatform, XcodeConfigIn } break; case MACOS: + case VISIONOS_DEVICE: + case VISIONOS_SIMULATOR: case WATCHOS_DEVICE: case WATCHOS_SIMULATOR: case TVOS_DEVICE: diff --git a/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeConfig.java b/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeConfig.java index 5df242200393a5..b2ed13fc09a22d 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeConfig.java +++ b/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeConfig.java @@ -126,6 +126,9 @@ public ConfiguredTarget create(RuleContext ruleContext) (appleOptions.iosMinimumOs != null) ? DottedVersion.maybeUnwrap(appleOptions.iosMinimumOs) : iosSdkVersion; + DottedVersion visionosSdkVersion = xcodeVersionProperties.getDefaultVisionosSdkVersion(); + // TODO: Replace with CppOptions.minimumOsVersion + DottedVersion visionosMinimumOsVersion = DottedVersion.fromStringUnchecked("1.0"); DottedVersion watchosSdkVersion = (appleOptions.watchOsSdkVersion != null) ? DottedVersion.maybeUnwrap(appleOptions.watchOsSdkVersion) @@ -155,6 +158,8 @@ public ConfiguredTarget create(RuleContext ruleContext) new XcodeConfigInfo( iosSdkVersion, iosMinimumOsVersion, + visionosSdkVersion, + visionosMinimumOsVersion, watchosSdkVersion, watchosMinimumOsVersion, tvosSdkVersion, diff --git a/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeConfigInfo.java b/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeConfigInfo.java index e5692368f3d940..d4ce9fbd2411d4 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeConfigInfo.java +++ b/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeConfigInfo.java @@ -39,6 +39,8 @@ public class XcodeConfigInfo extends NativeInfo private final DottedVersion iosSdkVersion; private final DottedVersion iosMinimumOsVersion; + private final DottedVersion visionosSdkVersion; + private final DottedVersion visionosMinimumOsVersion; private final DottedVersion watchosSdkVersion; private final DottedVersion watchosMinimumOsVersion; private final DottedVersion tvosSdkVersion; @@ -52,6 +54,8 @@ public class XcodeConfigInfo extends NativeInfo public XcodeConfigInfo( DottedVersion iosSdkVersion, DottedVersion iosMinimumOsVersion, + DottedVersion visionosSdkVersion, + DottedVersion visionosMinimumOsVersion, DottedVersion watchosSdkVersion, DottedVersion watchosMinimumOsVersion, DottedVersion tvosSdkVersion, @@ -64,6 +68,8 @@ public XcodeConfigInfo( boolean includeXcodeReqs) { this.iosSdkVersion = Preconditions.checkNotNull(iosSdkVersion); this.iosMinimumOsVersion = Preconditions.checkNotNull(iosMinimumOsVersion); + this.visionosSdkVersion = Preconditions.checkNotNull(visionosSdkVersion); + this.visionosMinimumOsVersion = Preconditions.checkNotNull(visionosMinimumOsVersion); this.watchosSdkVersion = Preconditions.checkNotNull(watchosSdkVersion); this.watchosMinimumOsVersion = Preconditions.checkNotNull(watchosMinimumOsVersion); this.tvosSdkVersion = Preconditions.checkNotNull(tvosSdkVersion); @@ -135,6 +141,8 @@ private XcodeConfigProvider() { public XcodeConfigInfoApi xcodeConfigInfo( String iosSdkVersion, String iosMinimumOsVersion, + String visionosSdkVersion, + String visionosMinimumOsVersion, String watchosSdkVersion, String watchosMinimumOsVersion, String tvosSdkVersion, @@ -147,6 +155,8 @@ private XcodeConfigProvider() { return new XcodeConfigInfo( DottedVersion.fromString(iosSdkVersion), DottedVersion.fromString(iosMinimumOsVersion), + DottedVersion.fromString(visionosSdkVersion), + DottedVersion.fromString(visionosMinimumOsVersion), DottedVersion.fromString(watchosSdkVersion), DottedVersion.fromString(watchosMinimumOsVersion), DottedVersion.fromString(tvosSdkVersion), @@ -195,6 +205,9 @@ public DottedVersion getMinimumOsForPlatformType(ApplePlatform.PlatformType plat return iosMinimumOsVersion; case TVOS: return tvosMinimumOsVersion; + case VISIONOS: + // TODO: Replace with CppOptions.minimumOsVersion + return DottedVersion.fromStringUnchecked("1.0"); case WATCHOS: return watchosMinimumOsVersion; case MACOS: @@ -216,6 +229,9 @@ public DottedVersion getSdkVersionForPlatform(ApplePlatform platform) { case TVOS_DEVICE: case TVOS_SIMULATOR: return tvosSdkVersion; + case VISIONOS_DEVICE: + case VISIONOS_SIMULATOR: + return visionosSdkVersion; case WATCHOS_DEVICE: case WATCHOS_SIMULATOR: return watchosSdkVersion; diff --git a/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeVersionProperties.java b/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeVersionProperties.java index f3e6e39e175423..22410e9887f0ab 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeVersionProperties.java +++ b/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeVersionProperties.java @@ -36,12 +36,14 @@ public class XcodeVersionProperties extends NativeInfo implements XcodePropertie new BuiltinProvider(STARLARK_NAME, XcodeVersionProperties.class) {}; @VisibleForTesting public static final String DEFAULT_IOS_SDK_VERSION = "8.4"; + @VisibleForTesting public static final String DEFAULT_VISIONOS_SDK_VERSION = "1.0"; @VisibleForTesting public static final String DEFAULT_WATCHOS_SDK_VERSION = "2.0"; @VisibleForTesting public static final String DEFAULT_MACOS_SDK_VERSION = "10.11"; @VisibleForTesting public static final String DEFAULT_TVOS_SDK_VERSION = "9.0"; private final Optional xcodeVersion; private final DottedVersion defaultIosSdkVersion; + private final DottedVersion defaultVisionosSdkVersion; private final DottedVersion defaultWatchosSdkVersion; private final DottedVersion defaultTvosSdkVersion; private final DottedVersion defaultMacosSdkVersion; @@ -63,7 +65,7 @@ public static XcodeVersionProperties unknownXcodeVersionProperties() { * specified. */ XcodeVersionProperties(DottedVersion xcodeVersion) { - this(xcodeVersion, null, null, null, null); + this(xcodeVersion, null, null, null, null, null); } /** @@ -73,6 +75,7 @@ public static XcodeVersionProperties unknownXcodeVersionProperties() { XcodeVersionProperties( DottedVersion xcodeVersion, @Nullable String defaultIosSdkVersion, + @Nullable String defaultVisionosSdkVersion, @Nullable String defaultWatchosSdkVersion, @Nullable String defaultTvosSdkVersion, @Nullable String defaultMacosSdkVersion) { @@ -81,6 +84,10 @@ public static XcodeVersionProperties unknownXcodeVersionProperties() { Strings.isNullOrEmpty(defaultIosSdkVersion) ? DottedVersion.fromStringUnchecked(DEFAULT_IOS_SDK_VERSION) : DottedVersion.fromStringUnchecked(defaultIosSdkVersion); + this.defaultVisionosSdkVersion = + Strings.isNullOrEmpty(defaultVisionosSdkVersion) + ? DottedVersion.fromStringUnchecked(DEFAULT_VISIONOS_SDK_VERSION) + : DottedVersion.fromStringUnchecked(defaultVisionosSdkVersion); this.defaultWatchosSdkVersion = Strings.isNullOrEmpty(defaultWatchosSdkVersion) ? DottedVersion.fromStringUnchecked(DEFAULT_WATCHOS_SDK_VERSION) @@ -117,6 +124,13 @@ public String getDefaultIosSdkVersionString() { return defaultIosSdkVersion != null ? defaultIosSdkVersion.toString() : null; } + /** Returns the default visionOS sdk version to use if this xcode version is in use. */ + @Nullable + @Override + public String getDefaultVisionosSdkVersionString() { + return defaultVisionosSdkVersion != null ? defaultVisionosSdkVersion.toString() : null; + } + /** Returns the default watchos sdk version to use if this xcode version is in use. */ @Nullable @Override @@ -148,6 +162,11 @@ public DottedVersion getDefaultIosSdkVersion() { return defaultIosSdkVersion; } + @Nullable + public DottedVersion getDefaultVisionosSdkVersion() { + return defaultVisionosSdkVersion; + } + @Nullable public DottedVersion getDefaultWatchosSdkVersion() { return defaultWatchosSdkVersion; @@ -174,6 +193,7 @@ public boolean equals(Object other) { XcodeVersionProperties otherData = (XcodeVersionProperties) other; return xcodeVersion.equals(otherData.getXcodeVersion()) && defaultIosSdkVersion.equals(otherData.getDefaultIosSdkVersion()) + && defaultVisionosSdkVersion.equals(otherData.getDefaultVisionosSdkVersion()) && defaultWatchosSdkVersion.equals(otherData.getDefaultWatchosSdkVersion()) && defaultTvosSdkVersion.equals(otherData.getDefaultTvosSdkVersion()) && defaultMacosSdkVersion.equals(otherData.getDefaultMacosSdkVersion()); @@ -184,6 +204,7 @@ public int hashCode() { return Objects.hash( xcodeVersion, defaultIosSdkVersion, + defaultVisionosSdkVersion, defaultWatchosSdkVersion, defaultTvosSdkVersion, defaultMacosSdkVersion); diff --git a/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeVersionRule.java b/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeVersionRule.java index 41609a36f41ef6..e813fa03d8b7b0 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeVersionRule.java +++ b/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeVersionRule.java @@ -31,6 +31,7 @@ public class XcodeVersionRule implements RuleDefinition { static final String VERSION_ATTR_NAME = "version"; static final String ALIASES_ATTR_NAME = "aliases"; static final String DEFAULT_IOS_SDK_VERSION_ATTR_NAME = "default_ios_sdk_version"; + static final String DEFAULT_VISIONOS_SDK_VERSION_ATTR_NAME = "default_visionos_sdk_version"; static final String DEFAULT_WATCHOS_SDK_VERSION_ATTR_NAME = "default_watchos_sdk_version"; static final String DEFAULT_TVOS_SDK_VERSION_ATTR_NAME = "default_tvos_sdk_version"; static final String DEFAULT_MACOS_SDK_VERSION_ATTR_NAME = "default_macos_sdk_version"; @@ -64,6 +65,13 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) .add( attr(DEFAULT_IOS_SDK_VERSION_ATTR_NAME, STRING) .nonconfigurable("this rule determines configuration")) + /* + The visionos sdk version that is used by default when this version of xcode is being used. + The visionos_sdk_version build flag will override the value specified here. + */ + .add( + attr(DEFAULT_VISIONOS_SDK_VERSION_ATTR_NAME, STRING) + .nonconfigurable("this rule determines configuration")) /* The watchos sdk version that is used by default when this version of xcode is being used. The watchos_sdk_version build flag will override the value specified here. diff --git a/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeVersionRuleData.java b/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeVersionRuleData.java index ff8cc3549c254a..266265911ac652 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeVersionRuleData.java +++ b/src/main/java/com/google/devtools/build/lib/rules/apple/XcodeVersionRuleData.java @@ -52,6 +52,8 @@ public class XcodeVersionRuleData implements TransitiveInfoProvider { attrMapper.get(XcodeVersionRule.VERSION_ATTR_NAME, Type.STRING)); String iosSdkVersionString = attrMapper.get(XcodeVersionRule.DEFAULT_IOS_SDK_VERSION_ATTR_NAME, Type.STRING); + String visionosSdkVersionString = + attrMapper.get(XcodeVersionRule.DEFAULT_VISIONOS_SDK_VERSION_ATTR_NAME, Type.STRING); String watchosSdkVersionString = attrMapper.get(XcodeVersionRule.DEFAULT_WATCHOS_SDK_VERSION_ATTR_NAME, Type.STRING); String tvosSdkVersionString = @@ -63,6 +65,7 @@ public class XcodeVersionRuleData implements TransitiveInfoProvider { new XcodeVersionProperties( xcodeVersion, iosSdkVersionString, + visionosSdkVersionString, watchosSdkVersionString, tvosSdkVersionString, macosxSdkVersionString); @@ -114,6 +117,9 @@ public boolean equals(Object other) { && xcodeVersionProperties .getDefaultIosSdkVersion() .equals(otherData.getXcodeVersionProperties().getDefaultIosSdkVersion()) + && xcodeVersionProperties + .getDefaultVisionosSdkVersion() + .equals(otherData.getXcodeVersionProperties().getDefaultVisionosSdkVersion()) && xcodeVersionProperties .getDefaultWatchosSdkVersion() .equals(otherData.getXcodeVersionProperties().getDefaultWatchosSdkVersion()) diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java index ed504331cffdf8..f991a687359a18 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/CompilationSupport.java @@ -134,7 +134,8 @@ public class CompilationSupport implements StarlarkValue { "-fexceptions", "-fasm-blocks", "-fobjc-abi-version=2", "-fobjc-legacy-dispatch"); /** - * Frameworks implicitly linked to iOS, watchOS, and tvOS binaries when using legacy compilation. + * Frameworks implicitly linked to iOS, visionOS, watchOS, and tvOS binaries when using legacy + * compilation. */ @VisibleForTesting static final NestedSet AUTOMATIC_SDK_FRAMEWORKS = diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/MultiArchBinarySupport.java b/src/main/java/com/google/devtools/build/lib/rules/objc/MultiArchBinarySupport.java index a407517c7f84fd..ff636f9cd0bf89 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/MultiArchBinarySupport.java +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/MultiArchBinarySupport.java @@ -251,6 +251,8 @@ private static ConfigurationDistinguisher configurationDistinguisher(PlatformTyp return ConfigurationDistinguisher.APPLEBIN_IOS; case CATALYST: return ConfigurationDistinguisher.APPLEBIN_CATALYST; + case VISIONOS: + return ConfigurationDistinguisher.APPLEBIN_VISIONOS; case WATCHOS: return ConfigurationDistinguisher.APPLEBIN_WATCHOS; case TVOS: @@ -286,6 +288,10 @@ private static DottedVersion.Option minimumOsVersionOption( case CATALYST: option = buildOptions.get(AppleCommandLineOptions.class).iosMinimumOs; break; + case VISIONOS: + // TODO: Replace with CppOptions.minimumOsVersion + option = DottedVersion.option(DottedVersion.fromStringUnchecked("1.0")); + break; case WATCHOS: option = buildOptions.get(AppleCommandLineOptions.class).watchosMinimumOs; break; @@ -333,6 +339,9 @@ private static BuildOptionsView defaultBuildOptionsForSplit( case MACOS: appleCommandLineOptions.macosMinimumOs = minimumOsVersionOption; break; + case VISIONOS: + // TODO: use CppOptions.minimumOsVersion + break; } return splitOptions; } @@ -461,6 +470,13 @@ public static ImmutableMap handleAppleCpus( } cpus = supportedAppleCpusFromMinimumOs(minimumOsVersionOption, cpus, platformType); break; + case VISIONOS: + cpus = buildOptions.get(AppleCommandLineOptions.class).visionosCpus; + if (cpus.isEmpty()) { + cpus = ImmutableList.of(AppleCommandLineOptions.DEFAULT_VISIONOS_CPU); + } + cpus = supportedAppleCpusFromMinimumOs(minimumOsVersionOption, cpus, platformType); + break; case WATCHOS: cpus = buildOptions.get(AppleCommandLineOptions.class).watchosCpus; if (cpus.isEmpty()) { diff --git a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcRuleClasses.java b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcRuleClasses.java index 4b3e044fd740a7..946b3fb728b27d 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcRuleClasses.java +++ b/src/main/java/com/google/devtools/build/lib/rules/objc/ObjcRuleClasses.java @@ -207,8 +207,8 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment envi return builder /* Names of SDK frameworks to link with (e.g. "AddressBook", "QuartzCore"). "UIKit" and - "Foundation" are always included when building for the iOS, tvOS and watchOS platforms. - For macOS, only "Foundation" is always included. + "Foundation" are always included when building for the iOS, tvOS, visionOS, + and watchOS platforms. For macOS, only "Foundation" is always included.

    When linking a top level Apple binary, all SDK frameworks listed in that binary's transitive dependency graph are linked. diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/apple/ApplePlatformTypeApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/apple/ApplePlatformTypeApi.java index e110ad94c78198..54edcfe31dd764 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/apple/ApplePlatformTypeApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/apple/ApplePlatformTypeApi.java @@ -23,7 +23,7 @@ name = "apple_platform_type", category = DocCategory.BUILTIN, doc = - "Describes an Apple \"platform type\", such as iOS, macOS, tvOS, or watchOS. This is" + "Describes an Apple \"platform type\", such as iOS, macOS, tvOS, visionOS, or watchOS. This is" + " distinct from a \"platform\", which is the platform type combined with one or more" + " CPU architectures.

    Specific instances of this type can be retrieved by accessing" + " the fields of the apple_common.platform_type.ios" + "

  • apple_common.platform_type.macos
    • " + "
  • apple_common.platform_type.tvos
    • " + + "
  • apple_common.platform_type.visionos
    • " + "
  • apple_common.platform_type.watchos

    • Likewise, the" + " platform type of an existing platform value can be retrieved using its" + " platform_type field.

    Platform types can be converted to a lowercase" diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/apple/XcodeConfigInfoApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/apple/XcodeConfigInfoApi.java index 781b5aa3b86160..8229f03ac5b423 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/apple/XcodeConfigInfoApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/apple/XcodeConfigInfoApi.java @@ -103,6 +103,16 @@ interface XcodeConfigProviderApi extends ProviderApi { named = true, positional = false, doc = "The ios minimum os version."), + @Param( + name = "visionosSdkVersion", + named = true, + positional = false, + doc = "The visionOS SDK version."), + @Param( + name = "visionosMinimumOsVersion", + named = true, + positional = false, + doc = "The visionOS minimum os version."), @Param( name = "watchosSdkVersion", named = true, @@ -144,6 +154,8 @@ interface XcodeConfigProviderApi extends ProviderApi { XcodeConfigInfoApi xcodeConfigInfo( String iosSdkVersion, String iosMinimumOsVersion, + String visionosSdkVersion, + String visionosMinimumOsVersion, String watchosSdkVersion, String watchosMinimumOsVersion, String tvosSdkVersion, diff --git a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/apple/XcodePropertiesApi.java b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/apple/XcodePropertiesApi.java index 4ad4b565dbe5f8..bedfbb42da2d7b 100644 --- a/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/apple/XcodePropertiesApi.java +++ b/src/main/java/com/google/devtools/build/lib/starlarkbuildapi/apple/XcodePropertiesApi.java @@ -45,6 +45,16 @@ public interface XcodePropertiesApi extends StructApi { @Nullable String getDefaultIosSdkVersionString(); + @StarlarkMethod( + name = "default_visionos_sdk_version", + doc = + "The default visionOS sdk version for this version of xcode, or None if " + + "unknown.", + structField = true, + allowReturnNones = true) + @Nullable + String getDefaultVisionosSdkVersionString(); + @StarlarkMethod( name = "default_watchos_sdk_version", doc = diff --git a/src/main/starlark/builtins_bzl/common/cc/cc_binary.bzl b/src/main/starlark/builtins_bzl/common/cc/cc_binary.bzl index 4beca92ef650c1..a657498cff4842 100644 --- a/src/main/starlark/builtins_bzl/common/cc/cc_binary.bzl +++ b/src/main/starlark/builtins_bzl/common/cc/cc_binary.bzl @@ -33,6 +33,8 @@ _LINKING_STATIC = "static_linking_mode" _IOS_SIMULATOR_TARGET_CPUS = ["ios_x86_64", "ios_i386", "ios_sim_arm64"] _IOS_DEVICE_TARGET_CPUS = ["ios_armv6", "ios_arm64", "ios_armv7", "ios_armv7s", "ios_arm64e"] +_VISIONOS_SIMULATOR_TARGET_CPUS = ["visionos_x86_64", "visionos_sim_arm64"] +_VISIONOS_DEVICE_TARGET_CPUS = ["visionos_arm64"] _WATCHOS_SIMULATOR_TARGET_CPUS = ["watchos_i386", "watchos_x86_64", "watchos_arm64"] _WATCHOS_DEVICE_TARGET_CPUS = ["watchos_armv7k", "watchos_arm64_32"] _TVOS_SIMULATOR_TARGET_CPUS = ["tvos_x86_64", "tvos_sim_arm64"] @@ -555,7 +557,7 @@ def _report_invalid_options(ctx, cc_toolchain, cpp_config): fail("The selected toolchain does not support setting --grte_top (it doesn't specify builtin_sysroot).") def _is_apple_platform(target_cpu): - if target_cpu in _IOS_SIMULATOR_TARGET_CPUS or target_cpu in _IOS_DEVICE_TARGET_CPUS or target_cpu in _WATCHOS_SIMULATOR_TARGET_CPUS or target_cpu in _WATCHOS_DEVICE_TARGET_CPUS or target_cpu in _TVOS_SIMULATOR_TARGET_CPUS or target_cpu in _TVOS_DEVICE_TARGET_CPUS or target_cpu in _CATALYST_TARGET_CPUS or target_cpu in _MACOS_TARGET_CPUS: + if target_cpu in _IOS_SIMULATOR_TARGET_CPUS or target_cpu in _IOS_DEVICE_TARGET_CPUS or target_cpu in _VISIONOS_SIMULATOR_TARGET_CPUS or target_cpu in _VISIONOS_DEVICE_TARGET_CPUS or target_cpu in _WATCHOS_SIMULATOR_TARGET_CPUS or target_cpu in _WATCHOS_DEVICE_TARGET_CPUS or target_cpu in _TVOS_SIMULATOR_TARGET_CPUS or target_cpu in _TVOS_DEVICE_TARGET_CPUS or target_cpu in _CATALYST_TARGET_CPUS or target_cpu in _MACOS_TARGET_CPUS: return True return False diff --git a/src/main/starlark/builtins_bzl/common/objc/transitions.bzl b/src/main/starlark/builtins_bzl/common/objc/transitions.bzl index 88aec6da87282b..f886f5933b1448 100644 --- a/src/main/starlark/builtins_bzl/common/objc/transitions.bzl +++ b/src/main/starlark/builtins_bzl/common/objc/transitions.bzl @@ -37,6 +37,14 @@ def _determine_single_architecture(platform_type, settings): if cpu_value == "darwin_arm64": return "sim_arm64" return DEFAULT_IOS_CPU + if platform_type == VISIONOS: + cpus = settings["//command_line_option:visionos_cpus"] + if len(cpus) > 0: + return cpus[0] + cpu_value = settings["//command_line_option:cpu"] + if cpu_value == "darwin_arm64": + return "sim_arm64" + return DEFAULT_VISIONOS_CPU if platform_type == WATCHOS: watchos_cpus = settings["//command_line_option:watchos_cpus"] if len(watchos_cpus) == 0: @@ -64,13 +72,16 @@ def _determine_single_architecture(platform_type, settings): fail("ERROR: Unhandled platform type {}".format(platform_type)) IOS = "ios" +VISIONOS = "visionos" WATCHOS = "watchos" TVOS = "tvos" MACOS = "macos" CATALYST = "catalyst" IOS_CPU_PREFIX = "ios_" +VISIONOS_CPU_PREFIX = "visionos_" DARWIN_CPU_PREFIX = "darwin_" DEFAULT_IOS_CPU = "x86_64" +DEFAULT_VISIONOS_CPU = "x86_64" DEFAULT_WATCHOS_CPU = "i386" DEFAULT_TVOS_CPU = "x86_64" DEFAULT_MACOS_CPU = "x86_64" @@ -122,6 +133,7 @@ _apple_rule_base_transition_inputs = [ "//command_line_option:ios_multi_cpus", "//command_line_option:macos_cpus", "//command_line_option:tvos_cpus", + "//command_line_option:visionos_cpus", "//command_line_option:watchos_cpus", "//command_line_option:catalyst_cpus", "//command_line_option:platforms", diff --git a/src/test/java/com/google/devtools/build/lib/packages/util/MockPlatformSupport.java b/src/test/java/com/google/devtools/build/lib/packages/util/MockPlatformSupport.java index ec3d6f2c7d315e..b657474765a2dc 100644 --- a/src/test/java/com/google/devtools/build/lib/packages/util/MockPlatformSupport.java +++ b/src/test/java/com/google/devtools/build/lib/packages/util/MockPlatformSupport.java @@ -112,6 +112,10 @@ public static void setup( " constraint_setting = ':os',", ")", "constraint_value(", + " name = 'visionos',", + " constraint_setting = ':os',", + ")", + "constraint_value(", " name = 'watchos',", " constraint_setting = ':os',", ")", diff --git a/src/test/java/com/google/devtools/build/lib/rules/apple/XcodeConfigTest.java b/src/test/java/com/google/devtools/build/lib/rules/apple/XcodeConfigTest.java index 51ea58bddd6337..cd41a807eb30f3 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/apple/XcodeConfigTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/apple/XcodeConfigTest.java @@ -445,7 +445,9 @@ public void xcodeVersionConfig_isFunction() throws Exception { + " tvosMinimumOsVersion='1.6'," + " macosSdkVersion='1.7'," + " macosMinimumOsVersion='1.8'," - + " xcodeVersion='1.9'))]", + + " visionosSdkVersion='1.9'," + + " visionosMinimumOsVersion='1.10'," + + " xcodeVersion='1.11'))]", "my_rule = rule(_impl, attrs = { 'dep' : attr.label() })"); scratch.file("foo/BUILD", "load(':extension.bzl', 'my_rule')", "my_rule(name='test')"); assertNoEvents(); @@ -466,6 +468,8 @@ public void xcodeVersionConfig_isFunction() throws Exception { DottedVersion.fromStringUnchecked("1.7"), DottedVersion.fromStringUnchecked("1.8"), DottedVersion.fromStringUnchecked("1.9"), + DottedVersion.fromStringUnchecked("1.10"), + DottedVersion.fromStringUnchecked("1.11"), XcodeConfigInfo.Availability.UNKNOWN, /** xcodeVersionFlagValue= */ "", @@ -489,7 +493,9 @@ public void xcodeVersionConfig_throwsOnBadInput() throws Exception { + " tvosMinimumOsVersion='1.6'," + " macosSdkVersion='1.7'," + " macosMinimumOsVersion='1.8'," - + " xcodeVersion='1.9'))]", + + " visionosSdkVersion='1.9'," + + " visionosMinimumOsVersion='1.10'," + + " xcodeVersion='1.11'))]", "my_rule = rule(_impl, attrs = { 'dep' : attr.label() })"); scratch.file("foo/BUILD", "load(':extension.bzl', 'my_rule')", "my_rule(name='test')"); assertNoEvents(); @@ -509,12 +515,14 @@ public void xcodeVersionConfig_exposesExpectedAttributes() throws Exception { + " iosSdkVersion='1.1'," + " iosMinimumOsVersion='1.2'," + " watchosSdkVersion='1.3'," - + " watchosMinimumOsVersion='1.4'," + + " watchosMinimumOsVersion='2.4'," + " tvosSdkVersion='1.5'," + " tvosMinimumOsVersion='1.6'," + " macosSdkVersion='1.7'," + " macosMinimumOsVersion='1.8'," - + " xcodeVersion='1.9')", + + " visionosSdkVersion='1.9'," + + " visionosMinimumOsVersion='1.10'," + + " xcodeVersion='1.11')", " return [result(xcode_version=xcode_version.xcode_version()," + "min_os=xcode_version.minimum_os_for_platform_type(ctx.fragments.apple.single_arch_platform.platform_type)),]", "my_rule = rule(_impl, attrs = { 'dep' : attr.label() }, fragments = ['apple'])"); @@ -525,7 +533,7 @@ public void xcodeVersionConfig_exposesExpectedAttributes() throws Exception { (StructImpl) myRuleTarget.get( new StarlarkProvider.Key(Label.parseCanonical("//foo:extension.bzl"), "result")); - assertThat(info.getValue("xcode_version").toString()).isEqualTo("1.9"); + assertThat(info.getValue("xcode_version").toString()).isEqualTo("1.11"); assertThat(info.getValue("min_os").toString()).isEqualTo("1.8"); } diff --git a/src/test/shell/bazel/android/android_integration_test.sh b/src/test/shell/bazel/android/android_integration_test.sh index a61fa6f461314e..776a40dbe0514d 100755 --- a/src/test/shell/bazel/android/android_integration_test.sh +++ b/src/test/shell/bazel/android/android_integration_test.sh @@ -232,7 +232,7 @@ android_binary( EOF cat > MODULE.bazel << 'EOF' # Required for android_integration_test_with_platforms -bazel_dep(name = "platforms", version = "0.0.5") +bazel_dep(name = "platforms", version = "0.0.7") EOF bazel clean diff --git a/tools/osx/BUILD b/tools/osx/BUILD index 79344650445338..2a0f44224ce2f2 100644 --- a/tools/osx/BUILD +++ b/tools/osx/BUILD @@ -3,6 +3,7 @@ load( "ios_sdk_version_flag", "macos_sdk_version_flag", "tvos_sdk_version_flag", + "visionos_sdk_version_flag", "watchos_sdk_version_flag", "xcode_version_flag", ) @@ -100,6 +101,8 @@ ios_sdk_version_flag(name = "ios_sdk_version_flag") tvos_sdk_version_flag(name = "tvos_sdk_version_flag") +visionos_sdk_version_flag(name = "visionos_sdk_version_flag") + watchos_sdk_version_flag(name = "watchos_sdk_version_flag") macos_sdk_version_flag(name = "macos_sdk_version_flag") diff --git a/tools/osx/xcode_configure.bzl b/tools/osx/xcode_configure.bzl index e8b7499a814861..e9c4152b924c01 100644 --- a/tools/osx/xcode_configure.bzl +++ b/tools/osx/xcode_configure.bzl @@ -71,6 +71,7 @@ def _xcode_version_output(repository_ctx, name, version, aliases, developer_dir, ios_sdk_version = _search_sdk_output(xcodebuild_result.stdout, "iphoneos") tvos_sdk_version = _search_sdk_output(xcodebuild_result.stdout, "appletvos") macos_sdk_version = _search_sdk_output(xcodebuild_result.stdout, "macosx") + visionos_sdk_version = _search_sdk_output(xcodebuild_result.stdout, "xros") watchos_sdk_version = _search_sdk_output(xcodebuild_result.stdout, "watchos") build_contents += "xcode_version(\n name = '%s'," % name build_contents += "\n version = '%s'," % version @@ -82,6 +83,8 @@ def _xcode_version_output(repository_ctx, name, version, aliases, developer_dir, build_contents += "\n default_tvos_sdk_version = '%s'," % tvos_sdk_version if macos_sdk_version: build_contents += "\n default_macos_sdk_version = '%s'," % macos_sdk_version + if visionos_sdk_version: + build_contents += "\n default_visionos_sdk_version = '%s'," % visionos_sdk_version if watchos_sdk_version: build_contents += "\n default_watchos_sdk_version = '%s'," % watchos_sdk_version build_contents += "\n)\n" diff --git a/tools/osx/xcode_version_flag.bzl b/tools/osx/xcode_version_flag.bzl index a7ede5f54dd95a..1578a731b8fe32 100644 --- a/tools/osx/xcode_version_flag.bzl +++ b/tools/osx/xcode_version_flag.bzl @@ -98,6 +98,16 @@ def _tvos_sdk_version_flag_impl(ctx): ), )) +def _visionos_sdk_version_flag_impl(ctx): + """A rule that allows select() to select based on the visionOS SDK version.""" + xcode_config = ctx.attr._xcode_config[apple_common.XcodeVersionConfig] + + return config_common.FeatureFlagInfo(value = _strip_version( + xcode_config.sdk_version_for_platform( + apple_common.platform.visionos_device, + ), + )) + def _watchos_sdk_version_flag_impl(ctx): """A rule that allows select() to select based on the watchOS SDK version.""" xcode_config = ctx.attr._xcode_config[apple_common.XcodeVersionConfig] @@ -200,6 +210,16 @@ tvos_sdk_version_flag = rule( }, ) +visionos_sdk_version_flag = rule( + implementation = _visionos_sdk_version_flag_impl, + attrs = { + "_xcode_config": attr.label(default = configuration_field( + fragment = "apple", + name = "xcode_config_label", + )), + }, +) + watchos_sdk_version_flag = rule( implementation = _watchos_sdk_version_flag_impl, attrs = { From bc8bb956c735b339dec50078142c290cde2de7f9 Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Mon, 11 Sep 2023 23:27:22 +0200 Subject: [PATCH 058/125] [6.4.0] Intern empty `Depset`s (#19443) Empty `Depset`s are interned per order, which decreases allocations as well as retained heap when a `Depset` is stored in a `struct` in a provider (providers with schema already unwrap most `Depset`s). Fixes #19380 Closes #19387. PiperOrigin-RevId: 563104923 Change-Id: If66fb4a108ef7569a4f95e7af3a74ac84d1c4636 --- .../build/lib/collect/nestedset/Depset.java | 14 ++++++++------ .../build/lib/collect/nestedset/Order.java | 9 ++++++++- .../build/lib/collect/nestedset/DepsetTest.java | 13 +++++++++++++ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/Depset.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/Depset.java index b013822bdf49e6..332092a04e0b39 100644 --- a/src/main/java/com/google/devtools/build/lib/collect/nestedset/Depset.java +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/Depset.java @@ -95,7 +95,7 @@ public final class Depset implements StarlarkValue, Debug.ValueWithDebugAttribut private final ElementType elemType; private final NestedSet set; - private Depset(ElementType elemType, NestedSet set) { + Depset(ElementType elemType, NestedSet set) { this.elemType = Preconditions.checkNotNull(elemType, "element type cannot be null"); this.set = set; } @@ -189,6 +189,9 @@ private static void checkElement(Object x, boolean strict) throws EvalException // two arguments: of(Class elemType, NestedSet set). We could also avoid the allocations // done by ElementType.of(). public static Depset of(ElementType elemType, NestedSet set) { + if (set.isEmpty()) { + return set.getOrder().emptyDepset(); + } return new Depset(elemType, set); } @@ -280,15 +283,11 @@ public static NestedSet cast(Object x, Class type, String what) throws public static NestedSet noneableCast(Object x, Class type, String what) throws EvalException { if (x == Starlark.NONE) { - @SuppressWarnings("unchecked") - NestedSet empty = (NestedSet) EMPTY; - return empty; + return NestedSetBuilder.emptySet(Order.STABLE_ORDER); } return cast(x, type, what); } - private static final NestedSet EMPTY = NestedSetBuilder.emptySet(Order.STABLE_ORDER); - public boolean isEmpty() { return set.isEmpty(); } @@ -390,6 +389,9 @@ static Depset fromDirectAndTransitive( } } + if (builder.isEmpty()) { + return builder.getOrder().emptyDepset(); + } return new Depset(type, builder.build()); } diff --git a/src/main/java/com/google/devtools/build/lib/collect/nestedset/Order.java b/src/main/java/com/google/devtools/build/lib/collect/nestedset/Order.java index 1a62d054fba06c..3f0ea09422ad4c 100644 --- a/src/main/java/com/google/devtools/build/lib/collect/nestedset/Order.java +++ b/src/main/java/com/google/devtools/build/lib/collect/nestedset/Order.java @@ -112,10 +112,12 @@ public enum Order { private final String starlarkName; private final NestedSet emptySet; + private final Depset emptyDepset; - private Order(String starlarkName) { + Order(String starlarkName) { this.starlarkName = starlarkName; this.emptySet = new NestedSet<>(this); + this.emptyDepset = new Depset(Depset.ElementType.EMPTY, this.emptySet); } @SerializationConstant @AutoCodec.VisibleForSerialization @@ -150,6 +152,11 @@ NestedSet emptySet() { return (NestedSet) emptySet; } + /** Returns an empty depset of the given ordering. */ + Depset emptyDepset() { + return emptyDepset; + } + public String getStarlarkName() { return starlarkName; } diff --git a/src/test/java/com/google/devtools/build/lib/collect/nestedset/DepsetTest.java b/src/test/java/com/google/devtools/build/lib/collect/nestedset/DepsetTest.java index cc6d597833fdd0..a2139858beb8f6 100644 --- a/src/test/java/com/google/devtools/build/lib/collect/nestedset/DepsetTest.java +++ b/src/test/java/com/google/devtools/build/lib/collect/nestedset/DepsetTest.java @@ -404,4 +404,17 @@ ev.new Scenario() + " NoneType'", "depset(direct='hello')"); } + + @Test + public void testEmptyDepsetInternedPerOrder() throws Exception { + ev.exec( + "stable1 = depset()", + "stable2 = depset()", + "preorder1 = depset(order = 'preorder')", + "preorder2 = depset(order = 'preorder')"); + assertThat(ev.lookup("stable1")).isSameInstanceAs(ev.lookup("stable2")); + assertThat(ev.lookup("preorder1")).isSameInstanceAs(ev.lookup("preorder2")); + assertThat(ev.lookup("stable1")).isNotSameInstanceAs(ev.lookup("preorder1")); + assertThat(ev.lookup("stable2")).isNotSameInstanceAs(ev.lookup("preorder2")); + } } From f465abd94b416f58e5f15f9ac966c8a87b5c7b2f Mon Sep 17 00:00:00 2001 From: Ivo List Date: Tue, 12 Sep 2023 06:35:02 +0200 Subject: [PATCH 059/125] [6.4.0] Do not allow applicable_licenses on platform. (#19426) RELNOTES: `applicable_licenses` is no longer allowed on the `platform` rule. Additionally, `default_package_metadata` from any `package` rule will not be applied. PiperOrigin-RevId: 529808047 Change-Id: I7c2c682055bde6924d68858d38ef63f32457cdb9 Co-authored-by: Googler --- .../build/lib/rules/platform/PlatformBaseRule.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/rules/platform/PlatformBaseRule.java b/src/main/java/com/google/devtools/build/lib/rules/platform/PlatformBaseRule.java index cabc5774a25745..70ca5848e2352b 100644 --- a/src/main/java/com/google/devtools/build/lib/rules/platform/PlatformBaseRule.java +++ b/src/main/java/com/google/devtools/build/lib/rules/platform/PlatformBaseRule.java @@ -21,7 +21,6 @@ import com.google.devtools.build.lib.analysis.PlatformConfiguration; import com.google.devtools.build.lib.analysis.RuleDefinition; import com.google.devtools.build.lib.analysis.RuleDefinitionEnvironment; -import com.google.devtools.build.lib.packages.BuildType; import com.google.devtools.build.lib.packages.RuleClass; import com.google.devtools.build.lib.packages.RuleClass.ToolchainResolutionMode; import com.google.devtools.build.lib.packages.Type; @@ -42,13 +41,7 @@ public RuleClass build(RuleClass.Builder builder, RuleDefinitionEnvironment env) // No need to show up in ":all", etc. target patterns. .value(ImmutableList.of("manual")) .nonconfigurable("low-level attribute, used in platform configuration")) - .override( - // A platform is essentially a constant which is never linked into a target. - // This will, in a very hacky way, suppress picking up default_applicable_licenses - attr("applicable_licenses", BuildType.LABEL_LIST) - .value(ImmutableList.of()) - .allowedFileTypes() - .nonconfigurable("fundamental constant, used in platform configuration")) + .removeAttribute("applicable_licenses") .exemptFromConstraintChecking("this rule helps *define* a constraint") .useToolchainResolution(ToolchainResolutionMode.DISABLED) .removeAttribute("deps") From 0d70b76f37d19ecf3b77df0c1595ef16ca327050 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Tue, 12 Sep 2023 11:55:19 +0200 Subject: [PATCH 060/125] [6.4.0] Cherry pick Bzlmod fixes (#19494) Cherry-picked commits: - 15b1575cff9ad6e7f22d49cf7df2c7f2a3015f53 - acb38a619261379ef5a6b5ccd472072e273019a8 - 07e0d316a345a3cb2593f98525320590bbc56e30 --------- Co-authored-by: salma-samy --- .../bazel/bzlmod/BazelDepGraphFunction.java | 6 ++- .../bazel/bzlmod/BazelLockFileFunction.java | 27 ++++++++--- .../lib/bazel/bzlmod/BazelLockFileValue.java | 5 ++ .../bzlmod/SingleExtensionEvalFunction.java | 3 +- .../build/lib/bazel/repository/PatchUtil.java | 2 +- .../build/lib/vfs/FileSystemUtils.java | 25 +++++++--- .../bzlmod/BazelLockFileFunctionTest.java | 4 +- .../StarlarkRepositoryContextTest.java | 2 +- .../py/bazel/bzlmod/bazel_lockfile_test.py | 46 +++++++++++++++++-- 9 files changed, 96 insertions(+), 24 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java index 9e7c0a328e8356..30597019640d22 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java @@ -97,7 +97,8 @@ public SkyValue compute(SkyKey skyKey, Environment env) throw new BazelDepGraphFunctionException( ExternalDepsException.withMessage( Code.BAD_MODULE, - "Lock file is no longer up-to-date because: %s", + "Lock file is no longer up-to-date because: %s. " + + "Please run `bazel mod deps --lockfile_mode=update` to update your lockfile.", String.join(", ", diffLockfile)), Transience.PERSISTENT); } @@ -129,6 +130,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (!lockfileMode.equals(LockfileMode.OFF)) { BazelLockFileValue updateLockfile = lockfile.toBuilder() + .setLockFileVersion(BazelLockFileValue.LOCK_FILE_VERSION) .setModuleFileHash(root.getModuleFileHash()) .setFlags(flags) .setLocalOverrideHashes(localOverrideHashes) @@ -183,7 +185,7 @@ static BzlmodFlagsAndEnvVars getFlagsAndEnvVars(Environment env) throws Interrup ImmutableMap moduleOverrides = ModuleFileFunction.MODULE_OVERRIDES.get(env).entrySet().stream() .collect( - toImmutableMap(e -> e.getKey(), e -> ((LocalPathOverride) e.getValue()).getPath())); + toImmutableMap(Entry::getKey, e -> ((LocalPathOverride) e.getValue()).getPath())); ImmutableList yankedVersions = ImmutableList.copyOf(YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.get(env)); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java index bbe02d643f6cf4..763b3828492301 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java @@ -39,6 +39,8 @@ import com.google.gson.JsonSyntaxException; import java.io.FileNotFoundException; import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import javax.annotation.Nullable; /** Reads the contents of the lock file into its value. */ @@ -46,6 +48,9 @@ public class BazelLockFileFunction implements SkyFunction { public static final Precomputed LOCKFILE_MODE = new Precomputed<>("lockfile_mode"); + private static final Pattern LOCKFILE_VERSION_PATTERN = + Pattern.compile("\"lockFileVersion\":\\s*(\\d+)"); + private final Path rootDirectory; private static final BzlmodFlagsAndEnvVars EMPTY_FLAGS = @@ -95,13 +100,21 @@ public static BazelLockFileValue getLockfileValue(RootedPath lockfilePath) throw BazelLockFileValue bazelLockFileValue; try { String json = FileSystemUtils.readContent(lockfilePath.asPath(), UTF_8); - bazelLockFileValue = - GsonTypeAdapterUtil.createLockFileGson( - lockfilePath - .asPath() - .getParentDirectory() - .getRelative(LabelConstants.MODULE_DOT_BAZEL_FILE_NAME)) - .fromJson(json, BazelLockFileValue.class); + Matcher matcher = LOCKFILE_VERSION_PATTERN.matcher(json); + int version = matcher.find() ? Integer.parseInt(matcher.group(1)) : -1; + if (version == BazelLockFileValue.LOCK_FILE_VERSION) { + bazelLockFileValue = + GsonTypeAdapterUtil.createLockFileGson( + lockfilePath + .asPath() + .getParentDirectory() + .getRelative(LabelConstants.MODULE_DOT_BAZEL_FILE_NAME)) + .fromJson(json, BazelLockFileValue.class); + } else { + // This is an old version, needs to be updated + // Keep old version to recognize the problem in error mode + bazelLockFileValue = EMPTY_LOCKFILE.toBuilder().setLockFileVersion(version).build(); + } } catch (FileNotFoundException e) { bazelLockFileValue = EMPTY_LOCKFILE; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java index af422048956416..d588b81d8f410d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java @@ -92,6 +92,11 @@ public ImmutableList getModuleAndFlagsDiff( ImmutableMap localOverrideHashes, BzlmodFlagsAndEnvVars flags) { ImmutableList.Builder moduleDiff = new ImmutableList.Builder<>(); + if (getLockFileVersion() != BazelLockFileValue.LOCK_FILE_VERSION) { + return moduleDiff + .add("the version of the lockfile is not compatible with the current Bazel") + .build(); + } if (!moduleFileHash.equals(getModuleFileHash())) { moduleDiff.add("the root MODULE.bazel has been modified"); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java index f9e4b641a55b70..b1ecd75a57b210 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java @@ -288,7 +288,8 @@ private SingleExtensionEvalValue tryGettingValueFromLockFile( throw new SingleExtensionEvalFunctionException( ExternalDepsException.withMessage( Code.BAD_MODULE, - "Lock file is no longer up-to-date because: %s", + "Lock file is no longer up-to-date because: %s. " + + "Please run `bazel mod deps --lockfile_mode=update` to update your lockfile.", String.join(", ", extDiff)), Transience.PERSISTENT); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/PatchUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/PatchUtil.java index 10eb28f2e55b81..66477cb53982b7 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/PatchUtil.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/PatchUtil.java @@ -248,7 +248,7 @@ private static ImmutableList readFile(Path file) throws IOException { } private static void writeFile(Path file, List content) throws IOException { - FileSystemUtils.writeLinesAs(file, UTF_8, content); + FileSystemUtils.writeLinesAs(file, UTF_8, content, "\n"); } private static boolean getReadPermission(int permission) { diff --git a/src/main/java/com/google/devtools/build/lib/vfs/FileSystemUtils.java b/src/main/java/com/google/devtools/build/lib/vfs/FileSystemUtils.java index 71df26fa8fa06d..1a0479a83ca527 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/FileSystemUtils.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/FileSystemUtils.java @@ -708,8 +708,8 @@ public static void writeContent(Path outputFile, byte[] content) throws IOExcept } /** - * Writes lines to file using the given encoding, ending every line with a line break '\n' - * character. + * Writes lines to file using the given encoding, ending every line with a system specific line + * break character. */ @ThreadSafe // but not atomic public static void writeLinesAs(Path file, Charset charset, String... lines) throws IOException { @@ -717,8 +717,8 @@ public static void writeLinesAs(Path file, Charset charset, String... lines) thr } /** - * Writes lines to file using the given encoding, ending every line with a line break '\n' - * character. + * Writes lines to file using the given encoding, ending every line with a system specific line + * break character. */ @ThreadSafe // but not atomic public static void writeLinesAs(Path file, Charset charset, Iterable lines) @@ -728,17 +728,28 @@ public static void writeLinesAs(Path file, Charset charset, Iterable lin } /** - * Appends lines to file using the given encoding, ending every line with a line break '\n' + * Writes lines to file using the given encoding, ending every line with a given line break * character. */ @ThreadSafe // but not atomic + public static void writeLinesAs( + Path file, Charset charset, Iterable lines, String lineBreak) throws IOException { + file.getParentDirectory().createDirectoryAndParents(); + asByteSink(file).asCharSink(charset).writeLines(lines, lineBreak); + } + + /** + * Appends lines to file using the given encoding, ending every line with a system specific line + * break character. + */ + @ThreadSafe // but not atomic public static void appendLinesAs(Path file, Charset charset, String... lines) throws IOException { appendLinesAs(file, charset, Arrays.asList(lines)); } /** - * Appends lines to file using the given encoding, ending every line with a line break '\n' - * character. + * Appends lines to file using the given encoding, ending every line with a system specific line + * break character. */ @ThreadSafe // but not atomic public static void appendLinesAs(Path file, Charset charset, Iterable lines) diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java index 2b7a308b4a5a20..9774f7f797019e 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java @@ -430,7 +430,9 @@ public void invalidLockfileEmptyFile() throws Exception { fail(result.getError().toString()); } - scratch.overwriteFile(rootDirectory.getRelative("MODULE.bazel.lock").getPathString(), "{}"); + scratch.overwriteFile( + rootDirectory.getRelative("MODULE.bazel.lock").getPathString(), + "{\"lockFileVersion\": " + BazelLockFileValue.LOCK_FILE_VERSION + "}"); result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); if (!result.hasError()) { diff --git a/src/test/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContextTest.java b/src/test/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContextTest.java index 219ef733aa0e96..dede30ed8488b2 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContextTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryContextTest.java @@ -317,7 +317,7 @@ public void testPatch() throws Exception { context.createFile( context.path("my.patch"), "--- foo\n+++ foo\n" + ONE_LINE_PATCH, false, true, thread); context.patch(patchFile, StarlarkInt.of(0), thread); - testOutputFile(foo.getPath(), String.format("line one%nline two%n")); + testOutputFile(foo.getPath(), "line one\nline two\n"); } @Test diff --git a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py index ed6212e4170623..f2750e504985c4 100644 --- a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py +++ b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py @@ -207,7 +207,8 @@ def testLockfileErrorMode(self): 'ERROR: Error computing the main repository mapping: Lock file is' ' no longer up-to-date because: the root MODULE.bazel has been' ' modified, the value of --check_direct_dependencies flag has' - ' been modified' + ' been modified. Please run' + ' `bazel mod deps --lockfile_mode=update` to update your lockfile.' ), stderr, ) @@ -251,7 +252,8 @@ def testLocalOverrideWithErrorMode(self): ( 'ERROR: Error computing the main repository mapping: Lock file is' ' no longer up-to-date because: The MODULE.bazel file has changed' - ' for the overriden module: bar' + ' for the overriden module: bar. Please run' + ' `bazel mod deps --lockfile_mode=update` to update your lockfile.' ), stderr, ) @@ -502,7 +504,8 @@ def testUpdateModuleExtensionErrorMode(self): 'implementation of the extension ' "'ModuleExtensionId{bzlFileLabel=//:extension.bzl, " "extensionName=lockfile_ext, isolationKey=Optional.empty}' or one " - 'of its transitive .bzl files has changed' + 'of its transitive .bzl files has changed. Please run' + ' `bazel mod deps --lockfile_mode=update` to update your lockfile.' ), stderr, ) @@ -688,7 +691,8 @@ def testChangeEnvVariableInErrorMode(self): ' variables the extension' " 'ModuleExtensionId{bzlFileLabel=//:extension.bzl," " extensionName=lockfile_ext, isolationKey=Optional.empty}' depends" - ' on (or their values) have changed' + ' on (or their values) have changed. Please run' + ' `bazel mod deps --lockfile_mode=update` to update your lockfile.' ), stderr, ) @@ -735,6 +739,40 @@ def testModuleExtensionWithFile(self): _, _, stderr = self.RunBazel(['build', '@hello//:all']) self.assertIn('I have changed now!', ''.join(stderr)) + def testOldVersion(self): + self.ScratchFile('MODULE.bazel') + self.ScratchFile('BUILD', ['filegroup(name = "hello")']) + self.RunBazel(['build', '--nobuild', '//:all']) + + # Set version to old + with open('MODULE.bazel.lock', 'r') as json_file: + data = json.load(json_file) + data['lockFileVersion'] = 0 + with open('MODULE.bazel.lock', 'w') as json_file: + json.dump(data, json_file, indent=4) + + # Run in error mode + exit_code, _, stderr = self.RunBazel( + ['build', '--nobuild', '--lockfile_mode=error', '//:all'], + allow_failure=True, + ) + self.AssertExitCode(exit_code, 48, stderr) + self.assertIn( + ( + 'ERROR: Error computing the main repository mapping: Lock file is' + ' no longer up-to-date because: the version of the lockfile is not' + ' compatible with the current Bazel. Please run' + ' `bazel mod deps --lockfile_mode=update` to update your lockfile.' + ), + stderr, + ) + + # Run again with update + self.RunBazel(['build', '--nobuild', '//:all']) + with open('MODULE.bazel.lock', 'r') as json_file: + data = json.load(json_file) + self.assertEqual(data['lockFileVersion'], 1) + def testExtensionEvaluationDoesNotRerunOnChangedImports(self): """ From 75db6b1bbb3d6663b6557555cf79d65ba1a0aa2b Mon Sep 17 00:00:00 2001 From: "bazel.build machine account" <15028808+bazel-io@users.noreply.github.com> Date: Tue, 12 Sep 2023 14:49:55 -0400 Subject: [PATCH 061/125] [6.4.0] Optimize classpath pre-processing in java_stub_template.txt (#19491) The classpath pre-processing in this `java_stub_template.txt` loop: https://github.com/bazelbuild/bazel/blob/fcfcb929366dd3faac9643302b19c88bcf871ec6/src/main/java/com/google/devtools/build/lib/bazel/rules/java/java_stub_template.txt#L309 is slow for long classpaths. For example for classpaths with ~250,000 and ~700,000 entries the loop takes 28 and 50 seconds, respectively, on an intel MacBook. This change reduce the times to 1 second or less. Fixes #19480 Closes #19481. Commit https://github.com/bazelbuild/bazel/commit/4e8f0bdb4eca04844a86e3c7affc8aa85245e75a PiperOrigin-RevId: 564491123 Change-Id: Id4be898c3f800d5390dd8bf997535a5e71a76ba3 Co-authored-by: Roman Salvador --- .../bazel/rules/java/java_stub_template.txt | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/java_stub_template.txt b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/java_stub_template.txt index b3f5070751dfd1..16129cc14dc1b2 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/java_stub_template.txt +++ b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/java_stub_template.txt @@ -306,21 +306,24 @@ function create_and_run_classpath_jar() { OLDIFS="$IFS" IFS="${CLASSPATH_SEPARATOR}" # Use a custom separator for the loop. + current_dir=$(pwd) for path in ${CLASSPATH}; do # Loop through the characters of the path and convert characters that are # not alphanumeric nor -_.~/ to their 2-digit hexadecimal representation - local i c buff - local converted_path="" - - for ((i=0; i<${#path}; i++)); do - c=${path:$i:1} - case ${c} in - [-_.~/a-zA-Z0-9] ) buff=${c} ;; - * ) printf -v buff '%%%02x' "'$c'" - esac - converted_path+="${buff}" - done - path=${converted_path} + if [[ ! $path =~ ^[-_.~/a-zA-Z0-9]*$ ]]; then + local i c buff + local converted_path="" + + for ((i=0; i<${#path}; i++)); do + c=${path:$i:1} + case ${c} in + [-_.~/a-zA-Z0-9] ) buff=${c} ;; + * ) printf -v buff '%%%02x' "'$c'" + esac + converted_path+="${buff}" + done + path=${converted_path} + fi if is_windows; then path="file:/${path}" # e.g. "file:/C:/temp/foo.jar" @@ -328,7 +331,7 @@ function create_and_run_classpath_jar() { # If not absolute, qualify the path case "${path}" in /*) ;; # Already an absolute path - *) path="$(pwd)/${path}";; # Now qualified + *) path="${current_dir}/${path}";; # Now qualified esac path="file:${path}" # e.g. "file:/usr/local/foo.jar" fi From 484b305907cc4d3e9d66059a7b84fc7e5dbb2d21 Mon Sep 17 00:00:00 2001 From: Chi Wang Date: Wed, 13 Sep 2023 00:52:24 +0200 Subject: [PATCH 062/125] [6.4.0] Add output name to CacheNotFoundException (#19452) to make it easier to debug errors related to remote cache eviction. Fixes #10025. PiperOrigin-RevId: 563361764 Change-Id: I2c29cd521c7ec1293f770b28a5cdcdf356750396 --- .../lib/remote/RemoteActionInputFetcher.java | 3 +- .../build/lib/remote/RemoteCache.java | 82 +++++++++++++++++-- .../lib/remote/RemoteExecutionService.java | 7 +- .../RemoteRepositoryRemoteExecutor.java | 12 ++- .../devtools/build/lib/remote/common/BUILD | 2 + .../remote/common/CacheNotFoundException.java | 18 +++- 6 files changed, 106 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcher.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcher.java index 04d6fb042cadf0..b0c8faf9a0f9a5 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcher.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcher.java @@ -124,7 +124,8 @@ protected ListenableFuture doDownloadFile( downloadProgressReporter = new DownloadProgressReporter(NO_ACTION, "", 0); } - return remoteCache.downloadFile(context, tempPath, digest, downloadProgressReporter); + return remoteCache.downloadFile( + context, execPath.getPathString(), tempPath, digest, downloadProgressReporter); } public static class DownloadProgress implements FetchProgress { diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java index 2ad56b0239896c..63659815361a26 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java @@ -33,6 +33,7 @@ import com.google.common.util.concurrent.SettableFuture; import com.google.devtools.build.lib.concurrent.ThreadSafety; import com.google.devtools.build.lib.exec.SpawnProgressEvent; +import com.google.devtools.build.lib.remote.common.CacheNotFoundException; import com.google.devtools.build.lib.remote.common.LazyFileOutputStream; import com.google.devtools.build.lib.remote.common.OutputDigestMismatchException; import com.google.devtools.build.lib.remote.common.ProgressStatusListener; @@ -201,6 +202,11 @@ protected ListenableFuture uploadBlob( return RxFutures.toListenableFuture(upload); } + public ListenableFuture downloadBlob( + RemoteActionExecutionContext context, Digest digest) { + return downloadBlob(context, /* blobName= */ "", digest); + } + /** * Downloads a blob with content hash {@code digest} and stores its content in memory. * @@ -208,14 +214,22 @@ protected ListenableFuture uploadBlob( * the content is stored in the future's {@code byte[]}. */ public ListenableFuture downloadBlob( - RemoteActionExecutionContext context, Digest digest) { + RemoteActionExecutionContext context, String blobName, Digest digest) { if (digest.getSizeBytes() == 0) { return EMPTY_BYTES; } ByteArrayOutputStream bOut = new ByteArrayOutputStream((int) digest.getSizeBytes()); + var download = downloadBlob(context, blobName, digest, bOut); SettableFuture outerF = SettableFuture.create(); + outerF.addListener( + () -> { + if (outerF.isCancelled()) { + download.cancel(/* mayInterruptIfRunning= */ true); + } + }, + directExecutor()); Futures.addCallback( - cacheProtocol.downloadBlob(context, digest, bOut), + download, new FutureCallback() { @Override public void onSuccess(Void aVoid) { @@ -237,12 +251,37 @@ public void onFailure(Throwable t) { } private ListenableFuture downloadBlob( - RemoteActionExecutionContext context, Digest digest, OutputStream out) { + RemoteActionExecutionContext context, String blobName, Digest digest, OutputStream out) { if (digest.getSizeBytes() == 0) { return COMPLETED_SUCCESS; } + var download = cacheProtocol.downloadBlob(context, digest, out); + SettableFuture future = SettableFuture.create(); + future.addListener( + () -> { + if (future.isCancelled()) { + download.cancel(/* mayInterruptIfRunning= */ true); + } + }, + directExecutor()); + Futures.addCallback( + download, + new FutureCallback() { + @Override + public void onSuccess(Void result) { + future.set(result); + } - return cacheProtocol.downloadBlob(context, digest, out); + @Override + public void onFailure(Throwable t) { + if (t instanceof CacheNotFoundException) { + ((CacheNotFoundException) t).setFilename(blobName); + } + future.setException(t); + } + }, + directExecutor()); + return future; } /** A reporter that reports download progresses. */ @@ -315,6 +354,13 @@ public ListenableFuture downloadFile( throws IOException { SettableFuture outerF = SettableFuture.create(); ListenableFuture f = downloadFile(context, localPath, digest, reporter); + outerF.addListener( + () -> { + if (outerF.isCancelled()) { + f.cancel(/* mayInterruptIfRunning= */ true); + } + }, + directExecutor()); Futures.addCallback( f, new FutureCallback() { @@ -325,7 +371,10 @@ public void onSuccess(Void unused) { @Override public void onFailure(Throwable throwable) { - if (throwable instanceof OutputDigestMismatchException) { + if (throwable instanceof CacheNotFoundException) { + var cacheNotFoundException = (CacheNotFoundException) throwable; + cacheNotFoundException.setFilename(outputPath); + } else if (throwable instanceof OutputDigestMismatchException) { OutputDigestMismatchException e = ((OutputDigestMismatchException) throwable); e.setOutputPath(outputPath); e.setLocalPath(localPath); @@ -341,11 +390,16 @@ public void onFailure(Throwable throwable) { /** Downloads a file (that is not a directory). The content is fetched from the digest. */ public ListenableFuture downloadFile( RemoteActionExecutionContext context, Path path, Digest digest) throws IOException { - return downloadFile(context, path, digest, new DownloadProgressReporter(NO_ACTION, "", 0)); + return downloadFile( + context, + path.getPathString(), + path, + digest, + new DownloadProgressReporter(NO_ACTION, "", 0)); } /** Downloads a file (that is not a directory). The content is fetched from the digest. */ - public ListenableFuture downloadFile( + private ListenableFuture downloadFile( RemoteActionExecutionContext context, Path path, Digest digest, @@ -409,7 +463,12 @@ public final List> downloadOutErr( downloads.add(Futures.immediateFailedFuture(e)); } } else if (result.hasStdoutDigest()) { - downloads.add(downloadBlob(context, result.getStdoutDigest(), outErr.getOutputStream())); + downloads.add( + downloadBlob( + context, + /* blobName= */ "", + result.getStdoutDigest(), + outErr.getOutputStream())); } if (!result.getStderrRaw().isEmpty()) { try { @@ -419,7 +478,12 @@ public final List> downloadOutErr( downloads.add(Futures.immediateFailedFuture(e)); } } else if (result.hasStderrDigest()) { - downloads.add(downloadBlob(context, result.getStderrDigest(), outErr.getErrorStream())); + downloads.add( + downloadBlob( + context, + /* blobName= */ "", + result.getStderrDigest(), + outErr.getErrorStream())); } return downloads; } diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java index ecf8f50166b880..ebe6efc6cb8659 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java @@ -1003,10 +1003,11 @@ ActionResultMetadata parseActionResultMetadata( Map> dirMetadataDownloads = Maps.newHashMapWithExpectedSize(result.getOutputDirectoriesCount()); for (OutputDirectory dir : result.getOutputDirectories()) { + var outputPath = encodeBytestringUtf8(dir.getPath()); dirMetadataDownloads.put( - remotePathResolver.outputPathToLocalPath(encodeBytestringUtf8(dir.getPath())), + remotePathResolver.outputPathToLocalPath(outputPath), Futures.transformAsync( - remoteCache.downloadBlob(context, dir.getTreeDigest()), + remoteCache.downloadBlob(context, outputPath, dir.getTreeDigest()), (treeBytes) -> immediateFuture(Tree.parseFrom(treeBytes, ExtensionRegistry.getEmptyRegistry())), directExecutor())); @@ -1176,7 +1177,7 @@ public InMemoryOutput downloadOutputs(RemoteAction action, RemoteActionResult re try (SilentCloseable c = Profiler.instance().profile("Remote.downloadInMemoryOutput")) { if (inMemoryOutput != null) { ListenableFuture inMemoryOutputDownload = - remoteCache.downloadBlob(context, inMemoryOutputDigest); + remoteCache.downloadBlob(context, inMemoryOutputPath.getPathString(), inMemoryOutputDigest); waitForBulkTransfer( ImmutableList.of(inMemoryOutputDownload), /* cancelRemainingOnInterrupt= */ true); byte[] data = getFromFuture(inMemoryOutputDownload); diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteRepositoryRemoteExecutor.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteRepositoryRemoteExecutor.java index 3a731f72709be2..8fb79e208f4cc7 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteRepositoryRemoteExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteRepositoryRemoteExecutor.java @@ -86,14 +86,18 @@ private ExecutionResult downloadOutErr(RemoteActionExecutionContext context, Act if (!result.getStdoutRaw().isEmpty()) { stdout = result.getStdoutRaw().toByteArray(); } else if (result.hasStdoutDigest()) { - stdout = Utils.getFromFuture(remoteCache.downloadBlob(context, result.getStdoutDigest())); + stdout = + Utils.getFromFuture( + remoteCache.downloadBlob(context, "", result.getStdoutDigest())); } byte[] stderr = new byte[0]; if (!result.getStderrRaw().isEmpty()) { stderr = result.getStderrRaw().toByteArray(); } else if (result.hasStderrDigest()) { - stderr = Utils.getFromFuture(remoteCache.downloadBlob(context, result.getStderrDigest())); + stderr = + Utils.getFromFuture( + remoteCache.downloadBlob(context, "", result.getStderrDigest())); } return new ExecutionResult(result.getExitCode(), stdout, stderr); @@ -138,7 +142,7 @@ public ExecutionResult execute( platform, timeout, acceptCached, - /*salt=*/ null); + /* salt= */ null); Digest actionDigest = digestUtil.compute(action); ActionKey actionKey = new ActionKey(actionDigest); CachedActionResult cachedActionResult; @@ -158,7 +162,7 @@ public ExecutionResult execute( additionalInputs.put(actionDigest, action); additionalInputs.put(commandHash, command); - remoteCache.ensureInputsPresent(context, merkleTree, additionalInputs, /*force=*/ true); + remoteCache.ensureInputsPresent(context, merkleTree, additionalInputs, /* force= */ true); } try (SilentCloseable c = diff --git a/src/main/java/com/google/devtools/build/lib/remote/common/BUILD b/src/main/java/com/google/devtools/build/lib/remote/common/BUILD index 36e0a515df1eb5..62a6dc047ec21d 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/common/BUILD +++ b/src/main/java/com/google/devtools/build/lib/remote/common/BUILD @@ -14,6 +14,8 @@ java_library( name = "cache_not_found_exception", srcs = ["CacheNotFoundException.java"], deps = [ + "//third_party:guava", + "//third_party:jsr305", "@remoteapis//:build_bazel_remote_execution_v2_remote_execution_java_proto", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/remote/common/CacheNotFoundException.java b/src/main/java/com/google/devtools/build/lib/remote/common/CacheNotFoundException.java index cad87a15023875..b48ccb2b23fa9b 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/common/CacheNotFoundException.java +++ b/src/main/java/com/google/devtools/build/lib/remote/common/CacheNotFoundException.java @@ -15,7 +15,9 @@ package com.google.devtools.build.lib.remote.common; import build.bazel.remote.execution.v2.Digest; +import com.google.common.base.Strings; import java.io.IOException; +import javax.annotation.Nullable; /** * An exception to indicate cache misses. TODO(olaola): have a class of checked @@ -23,13 +25,27 @@ */ public final class CacheNotFoundException extends IOException { private final Digest missingDigest; + @Nullable private String filename; public CacheNotFoundException(Digest missingDigest) { - super("Missing digest: " + missingDigest.getHash() + "/" + missingDigest.getSizeBytes()); this.missingDigest = missingDigest; } + public void setFilename(@Nullable String filename) { + this.filename = filename; + } + public Digest getMissingDigest() { return missingDigest; } + + @Override + public String getMessage() { + String message = + "Missing digest: " + missingDigest.getHash() + "/" + missingDigest.getSizeBytes(); + if (!Strings.isNullOrEmpty(filename)) { + message += " for " + filename; + } + return message; + } } From 842282ecd64e3c41005a451704888263156fe58b Mon Sep 17 00:00:00 2001 From: Greg Date: Tue, 12 Sep 2023 19:44:44 -0400 Subject: [PATCH 063/125] [6.4.0] feat: add option to exit early if analysis cache is discarded (#19503) Forked from https://github.com/bazelbuild/bazel/pull/16805. Manually merged conflicts observed at https://github.com/bazelbuild/bazel/pull/16805#issuecomment-1664767974. Fixes https://github.com/bazelbuild/bazel/issues/19162. --------- Co-authored-by: Matt Mackay --- .../build/lib/analysis/AnalysisOptions.java | 11 +++++++++ .../build/lib/analysis/BuildView.java | 5 ++-- .../devtools/build/lib/buildeventstream/BUILD | 1 + .../BuildCompletingEvent.java | 24 +++++++++++++++---- .../proto/build_event_stream.proto | 3 +++ .../buildevent/BuildCompleteEvent.java | 2 +- .../build/lib/skyframe/SkyframeBuildView.java | 12 +++++++++- src/main/protobuf/failure_details.proto | 2 ++ .../lib/analysis/AnalysisCachingTest.java | 13 ++++++++++ .../google/devtools/build/lib/analysis/BUILD | 1 + .../analysis/util/BuildViewForTesting.java | 12 ++++++++-- 11 files changed, 76 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisOptions.java b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisOptions.java index 95256a0549fd24..6810d1f0525816 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/AnalysisOptions.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/AnalysisOptions.java @@ -39,6 +39,17 @@ public class AnalysisOptions extends OptionsBase { ) public boolean discardAnalysisCache; + @Option( + name = "allow_analysis_cache_discard", + defaultValue = "true", + documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, + effectTags = {OptionEffectTag.EAGERNESS_TO_EXIT}, + help = + "If discarding the analysis cache due to a change in the build system, setting this" + + " option to false will cause bazel to exit, rather than continuing with the build." + + " This option has no effect when 'discard_analysis_cache' is also set.") + public boolean allowAnalysisCacheDiscards; + @Option( name = "max_config_changes_to_show", defaultValue = "3", diff --git a/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java b/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java index c4208b0a9325c0..14a955d0b205a5 100644 --- a/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java +++ b/src/main/java/com/google/devtools/build/lib/analysis/BuildView.java @@ -256,7 +256,8 @@ public AnalysisResult update( } skyframeBuildView.setConfigurations( - eventHandler, configurations, viewOptions.maxConfigChangesToShow); + eventHandler, configurations, viewOptions.maxConfigChangesToShow, + viewOptions.allowAnalysisCacheDiscards); eventBus.post( new MakeEnvironmentEvent(configurations.getTargetConfiguration().getMakeEnvironment())); @@ -535,7 +536,7 @@ private AnalysisResult createResult( TopLevelTargetsAndConfigsResult topLevelTargetsWithConfigs, boolean includeExecutionPhase) throws InterruptedException { - Set