Merge upstream changes from repo https://chromium.googlesource.com/in…

…fra/infra branch main

3pp/nsjail/3pp.pb

@@ -7,9 +7,9 @@ create {
       repo: "https://github.com/google/nsjail"
       tag_pattern: "%s"
 
-      # We would like to use a fixed version of nsjail so that we can keep a
-      # copy of its config proto in our codebase. Fixed to 3.0 for now.
-      version_restriction: { op: EQ val: "3.0"}
+      # We would like to use a fixed version of nsjail so that we can keep
+      # its config stable in our codebase. Fixed to 3.1 for now.
+      version_restriction: { op: EQ val: "3.1"}
     }
     patch_dir: "patches"
     patch_version: "chromium.1"

DEPS

@@ -26,7 +26,7 @@ deps = {
 
   "infra/luci":
      "{chromium_git}/infra/luci/luci-py@" +
-     "bd51a9097394ef662736f9ec0ea6cfeb4c7bd09a",
+     "cbe8b4ba151ffd04402136208185d5b89415dab3",
 
   # This unpinned dependency is present because it is used by the trybots for
   # the recipes-py repo; They check out infra with this at HEAD, and then apply
@@ -39,15 +39,15 @@ deps = {
 
   "infra/go/src/go.chromium.org/luci":
      "{chromium_git}/infra/luci/luci-go@" +
-     "01cb6ea19c643c03eb9fae45757b14c6f57ca343",
+     "9aeb7facb5bcc2ee82acc267f11f100bacc89063",
 
   "infra/go/src/go.chromium.org/chromiumos/config":
      "{chromium_git}/chromiumos/config@" +
-     "dad32402130e08783ed01aa67c69bfbd7c2a3d61",
+     "536b076e636f20d2d2be46a563057c625507df61",
 
   "infra/go/src/go.chromium.org/chromiumos/infra/proto":
      "{chromium_git}/chromiumos/infra/proto@" +
-     "67347c16c08bd5670ec528b9dac9767ad4b692f8",
+     "e1c4f7a820fca76133b443b65888e51c57576a49",
 
   # Appengine third_party DEPS
   "infra/appengine/third_party/bootstrap":

appengine/cr-buildbucket/config.py

@@ -29,14 +29,9 @@
 from go.chromium.org.luci.buildbucket.proto import service_config_pb2
 import errors
 
-CURRENT_BUCKET_SCHEMA_VERSION = 12
+CURRENT_BUCKET_SCHEMA_VERSION = 13
 ACL_SET_NAME_RE = re.compile('^[a-z0-9_]+$')
 
-# The default percentage of builds that are marked as canary.
-# This number is relatively high so we treat canary seriously and that we have
-# a strong signal if the canary is broken.
-_DEFAULT_CANARY_PERCENTAGE = 10
-
 # The memcache key for get_all_bucket_ids_async cache.
 _MEMCACHE_ALL_BUCKET_IDS_KEY = 'all_bucket_ids_v1'
 # Expiration time for get_all_bucket_ids_async cache.
@@ -118,12 +113,6 @@ def validate_buildbucket_cfg(cfg, ctx):
 
       validate_access_list(acl_set.acls, ctx)
 
-  mixin_ctx = validation.Context(  # pragma: no cover
-      on_message=lambda msg: ctx.msg(msg.severity, '%s', msg.text))
-  swarmingcfg.validate_builder_mixins(cfg.builder_mixins, mixin_ctx)
-  mixins_are_valid = not mixin_ctx.result().has_errors
-  mixin_by_name = {m.name: m for m in cfg.builder_mixins}
-
   bucket_names = set()
 
   for i, bucket in enumerate(cfg.buckets):
@@ -151,8 +140,7 @@ def validate_buildbucket_cfg(cfg, ctx):
       if bucket.HasField('swarming'):  # pragma: no cover
         with ctx.prefix('swarming: '):
           swarmingcfg.validate_project_cfg(
-              bucket.swarming, well_known_experiments, mixin_by_name,
-              mixins_are_valid, ctx
+              bucket.swarming, well_known_experiments, ctx
           )
 
 
@@ -295,7 +283,7 @@ def _use_memcache(cls, _):
     return False
 
   # Binary config content.
-  config = datastore_utils.ProtobufProperty(project_config_pb2.Builder)
+  config = datastore_utils.ProtobufProperty(project_config_pb2.BuilderConfig)
   # Hash used for fast deduplication of configs. Set automatically on put.
   config_hash = ndb.StringProperty(required=True)
 
@@ -475,11 +463,8 @@ def cron_update_buckets():
   """Synchronizes bucket entities with configs fetched from luci-config.
 
   When storing in the datastore, inlines the referenced ACL sets and clears
-  the acl_sets message field. Also inlines swarmbucket builder defaults and
-  mixins and clears Builder.mixins field.
+  the acl_sets message field.
   """
-  import flatten_swarmingcfg
-
   config_map = config.get_project_configs(
       cfg_path(), project_config_pb2.BuildbucketCfg
   )
@@ -507,7 +492,6 @@ def cron_update_buckets():
         project_cfg.SerializeToString(deterministic=True)
     ).hexdigest()
     acl_sets_by_name = {a.name: a for a in project_cfg.acl_sets}
-    builder_mixins_by_name = {m.name: m for m in project_cfg.builder_mixins}
 
     for bucket_cfg in project_cfg.buckets:
       bucket_key = Bucket.make_key(
@@ -540,31 +524,15 @@ def cron_update_buckets():
       builders_to_delete = set(builders)
       builders_to_put = []
       if bucket_cfg.HasField('swarming'):
-        # Pull builder defaults out and apply defaults.
-        defaults = bucket_cfg.swarming.builder_defaults
-        bucket_cfg.swarming.ClearField('builder_defaults')
-        if not any(d.startswith('pool:') for d in defaults.dimensions):
-          defaults.dimensions.append(
-              'pool:luci.%s.%s' %
-              (project_id, short_bucket_name(bucket_cfg.name))
-          )
-        defaults.swarming_host = (
-            defaults.swarming_host or bucket_cfg.swarming.hostname
-        )
-
-        f = 'task_template_canary_percentage'
-        if not defaults.HasField(f) and bucket_cfg.swarming.HasField(f):
-          defaults.task_template_canary_percentage.CopyFrom(
-              bucket_cfg.swarming.task_template_canary_percentage
-          )
-
         for b in bucket_cfg.swarming.builders:
-          flatten_swarmingcfg.flatten_builder(
-              b, defaults, builder_mixins_by_name
-          )
           builder_key = Builder.make_key(project_id, bucket_key.id(), b.name)
           builders_to_delete.discard(builder_key)
           builder = builders.get(builder_key)
+          if not any(d.startswith('pool:') for d in b.dimensions):
+            b.dimensions.append(
+                'pool:luci.%s.%s' %
+                (project_id, short_bucket_name(bucket_cfg.name))
+            )
           if builder and builder.config_hash == Builder.compute_hash(b):
             continue
           builders_to_put.append(b)
@@ -643,7 +611,7 @@ def builder_matches(builder_id_msg, predicate):
   """Returns True iff builder_id_msg matches the predicate.
 
   Args:
-    * builder_id_msg (builder_pb2.BuilderID)
+    * builder_id_msg (builder_common_pb2.BuilderID)
     * predicate (service_config_pb2.BuilderPredicate)
   """
   builder_str = builder_id_string(builder_id_msg)

appengine/cr-buildbucket/creation.py

@@ -28,7 +28,6 @@
 import buildtags
 import config
 import experiments
-import flatten_swarmingcfg
 import errors
 import events
 import model
@@ -91,7 +90,7 @@ def __new__(
       retry_of (int): value for model.Build.retry_of attribute.
       pubsub_callback_auth_token (str): value for
         model.Build.pubsub_callback.auth_token. Allowed iff r.notify is set.
-      override_builder_cfg: a function (project_config_pb2.Builder) => None
+      override_builder_cfg: a func (project_config_pb2.BuilderConfig) => None
         that may modify the config in-place before deriving a build from it.
     """
     assert schedule_build_request
@@ -238,7 +237,8 @@ def compute_tag_set(sbr):
   def compute_experiments(sbr, builder_cfg, settings):
     """Returns a Dict[str, bool] of enabled/disabled experiments."""
     global_exps = []
-    ignored_exps = []
+    # not enable BBAGENT_DOWNLOAD_CIPD for swarmbucket or v1 related flow.
+    ignored_exps = [experiments.BBAGENT_DOWNLOAD_CIPD]
 
     for exp in settings.experiment.experiments:
       if exp.inactive:
@@ -385,7 +385,9 @@ class NewBuild(object):
 
   def __init__(self, request, builder_cfg):
     assert isinstance(request, BuildRequest)
-    assert isinstance(builder_cfg, (type(None), project_config_pb2.Builder))
+    assert isinstance(
+        builder_cfg, (type(None), project_config_pb2.BuilderConfig)
+    )
     self.request = request
     self.builder_cfg = builder_cfg
 
@@ -663,9 +665,22 @@ def _apply_global_settings(settings, build_proto):
     _add_configured_cache(build_proto, c)
 
 
+def _read_properties(recipe):
+  """Parses build properties from the recipe message.
+
+  Expects the message to be valid.
+  """
+  result = dict(p.split(':', 1) for p in recipe.properties)
+  for p in recipe.properties_j:  # pragma: no cover
+    k, v = p.split(':', 1)
+    parsed = json.loads(v)
+    result[k] = parsed
+  return result
+
+
 @ndb.tasklet
 def _apply_builder_config_async(builder_cfg, build_proto, exps):
-  """Applies project_config_pb2.Builder to a builds_pb2.Build."""
+  """Applies project_config_pb2.BuilderConfig to a builds_pb2.Build."""
   # Populate timeouts.
   build_proto.scheduling_timeout.seconds = builder_cfg.expiration_secs
   if not build_proto.scheduling_timeout.seconds:
@@ -694,9 +709,7 @@ def _apply_builder_config_async(builder_cfg, build_proto, exps):
     build_proto.exe.cipd_version = (
         builder_cfg.recipe.cipd_version or 'refs/heads/master'
     )
-    build_proto.input.properties.update(
-        flatten_swarmingcfg.read_properties(builder_cfg.recipe)
-    )
+    build_proto.input.properties.update(_read_properties(builder_cfg.recipe))
     build_proto.input.properties['recipe'] = builder_cfg.recipe.name
     build_proto.infra.recipe.cipd_package = builder_cfg.recipe.cipd_package
     build_proto.infra.recipe.name = builder_cfg.recipe.name

appengine/cr-buildbucket/legacy/api.py

@@ -22,7 +22,7 @@
 import gae_ts_mon
 
 from legacy import api_common
-from go.chromium.org.luci.buildbucket.proto import builder_pb2
+from go.chromium.org.luci.buildbucket.proto import builder_common_pb2
 from go.chromium.org.luci.buildbucket.proto import builds_service_pb2 as rpc_pb2
 from go.chromium.org.luci.buildbucket.proto import common_pb2
 from go.chromium.org.luci.buildbucket.proto import project_config_pb2
@@ -31,7 +31,6 @@
 import config
 import creation
 import errors
-import flatten_swarmingcfg
 import model
 import search
 import service
@@ -165,36 +164,7 @@ def assert_object(name, value):
     assert_object('swarming', swarming)
     swarming = copy.deepcopy(swarming)
 
-    override_builder_cfg_data = swarming.pop('override_builder_cfg', None)
-    if override_builder_cfg_data is not None:
-      assert_object('swarming.override_builder_cfg', override_builder_cfg_data)
-      if 'build_numbers' in override_builder_cfg_data:
-        bad(
-            'swarming.override_builder_cfg parameter '
-            'cannot override build_numbers'
-        )
-
-      override_builder_cfg = project_config_pb2.Builder()
-      try:
-        protoutil.merge_dict(override_builder_cfg_data, override_builder_cfg)
-      except TypeError as ex:
-        bad('swarming.override_builder_cfg parameter: %s', ex)
-      if override_builder_cfg.name:
-        bad('swarming.override_builder_cfg cannot override builder name')
-      if override_builder_cfg.mixins:
-        bad('swarming.override_builder_cfg cannot use mixins')
-      if any(d.startswith('pool:') for d in override_builder_cfg.dimensions):
-        logging.warning(
-            'pool is being overridden: %s', override_builder_cfg.dimensions
-        )
-      if 'pool:' in override_builder_cfg.dimensions:
-        bad('swarming.override_builder_cfg cannot remove pool dimension')
-      with ctx.prefix('swarming.override_builder_cfg parameter: '):
-        swarmingcfg.validate_builder_cfg(
-            override_builder_cfg, set(), [], False, ctx
-        )
-
-    if swarming:
+    if swarming:  # pragma: no cover
       bad('unrecognized keys in swarming param: %r', swarming.keys())
 
   properties = params.get('properties')
@@ -251,7 +221,7 @@ def put_request_message_to_build_request(put_request, well_known_experiments):
 
   # Create a v2 request.
   sbr = rpc_pb2.ScheduleBuildRequest(
-      builder=builder_pb2.BuilderID(builder=builder),
+      builder=builder_common_pb2.BuilderID(builder=builder),
       properties=bbutil.dict_to_struct(props),
       request_id=put_request.client_operation_id,
       experimental=bbutil.BOOLISH_TO_TRINARY[put_request.experimental],
@@ -317,51 +287,9 @@ def put_request_message_to_build_request(put_request, well_known_experiments):
       parameters=parameters,
       lease_expiration_date=lease_expiration_date,
       pubsub_callback_auth_token=pubsub_callback_auth_token,
-      override_builder_cfg=_override_builder_cfg_func(parameters),
   )
 
 
-def _override_builder_cfg_func(parameters):
-  """Returns a function that overrides a Builder config.
-
-  May return None.
-
-  See creation.BuildRequest.override_builder_cfg.
-  """
-  swarming_param = parameters.get(_PARAM_SWARMING) or {}
-  overrides_dict = swarming_param.get('override_builder_cfg')
-  if not overrides_dict:
-    return None
-
-  base_overrides = project_config_pb2.Builder()
-  protoutil.merge_dict(overrides_dict, base_overrides)
-
-  def _merge(cfg):
-    over = project_config_pb2.Builder()
-    over.CopyFrom(base_overrides)
-
-    # handle the case where the caller is updating the old recipe message but
-    # the `cfg` uses the new `exe` field.
-    if cfg.HasField('exe') and over.HasField('recipe'):
-      exe, recipe = over.exe, over.recipe
-
-      new_props = json.loads(over.properties or '{}')
-      new_props.update(flatten_swarmingcfg.read_properties(recipe))
-
-      if recipe.name:
-        new_props['recipe'] = recipe.name
-
-      exe.cipd_package = recipe.cipd_package
-      exe.cipd_version = recipe.cipd_version
-
-      over.ClearField('recipe')
-      over.properties = json.dumps(new_props)
-
-    return flatten_swarmingcfg.merge_builder(cfg, over)
-
-  return _merge
-
-
 def builds_to_messages(builds, include_lease_key=False):
   """Converts model.Build objects to BuildMessage objects.