Dt e3 0 update

Gemfile

@@ -12,6 +12,7 @@ gem 'coder'
 gem 'connection_pool'
 gem 'faraday'
 gem 'faraday_middleware'
+gem 'jemalloc', github: 'travis-ci/jemalloc-rb'
 gem 'jwt', '~> 1.5'
 gem 'metriks', '0.9.9.6'
 gem 'metriks-librato_metrics', github: 'eric/metriks-librato_metrics'
@@ -24,6 +25,7 @@ gem 'puma'
 gem 'rack', '>= 2.2.4'
 gem 'rack-ssl', '~> 1.4'
 gem 'rack-test', '>= 2.1.0'
+gem 'redis', '~> 4'
 gem 'rake'
 gem 'rbtrace'
 gem 'rerun', require: false, group: :development
@@ -34,7 +36,7 @@ gem 'simplecov', require: false, group: %i[development test]
 gem 'sinatra', '>= 3.0.6'
 gem 'travis'
 gem 'travis-config', github: 'travis-ci/travis-config'
-gem 'travis-github_apps', github: 'travis-ci/travis-github_apps'
+gem 'travis-github_apps', git: 'https://github.com/travis-ci/travis-github_apps', branch: 'ga-ext_access'
 gem 'travis-rollout', github: 'travis-ci/travis-rollout'
 gem 'travis-support', github: 'travis-ci/travis-support'
 

Gemfile.lock

@@ -18,6 +18,12 @@ GIT
       sinatra (= 3.0.6)
       tilt (~> 2.0)
 
+GIT
+  remote: https://github.com/travis-ci/jemalloc-rb.git
+  revision: e9528e26aff663efa2c124cdeca47700e389097d
+  specs:
+    jemalloc (1.0.2)
+
 GIT
   remote: https://github.com/travis-ci/travis-config.git
   revision: 601e9a6dcb799dc14944c40153c60aaf935629ee
@@ -26,8 +32,9 @@ GIT
       hashr (~> 2.0)
 
 GIT
-  remote: https://github.com/travis-ci/travis-github_apps.git
-  revision: c96dc9330849ff3e2ccd7c9d00005a1a96c1a4b6
+  remote: https://github.com/travis-ci/travis-github_apps
+  revision: 929dadf0b4f60ca4240d52a11fc032917ff3f83a
+  branch: ga-ext_access
   specs:
     travis-github_apps (0.2.1)
       activesupport (>= 3.2)
@@ -67,7 +74,7 @@ GEM
     coder (0.4.0)
     coderay (1.1.3)
     concurrent-ruby (1.2.2)
-    connection_pool (2.2.2)
+    connection_pool (2.4.1)
     crack (0.4.5)
       rexml
     diff-lcs (1.3)
@@ -158,7 +165,7 @@ GEM
       ffi (>= 1.0.6)
       msgpack (>= 0.4.3)
       optimist (>= 3.0.0)
-    redis (4.1.4)
+    redis (4.8.1)
     rerun (0.14.0)
       listen (~> 3.0)
     rest-client (2.1.0)
@@ -239,6 +246,7 @@ DEPENDENCIES
   connection_pool
   faraday
   faraday_middleware
+  jemalloc!
   jwt (~> 1.5)
   metriks (= 0.9.9.6)
   metriks-librato_metrics!
@@ -253,6 +261,7 @@ DEPENDENCIES
   rack-test (>= 2.1.0)
   rake
   rbtrace
+  redis (~> 4)
   rerun
   rest-client
   rspec (~> 3.0)

lib/travis/build/addons.rb

@@ -1,4 +1,4 @@
-require 'active_support/core_ext/string/inflections'
+require 'active_support/core_ext/string/inflections.rb'
 require 'travis/build/addons/apt'
 require 'travis/build/addons/apt_packages'
 require 'travis/build/addons/apt_retries'

lib/travis/build/appliances/agent.rb

@@ -37,7 +37,7 @@ def start
           end
 
           def store_key
-            redis.set(key, 1, ex: TTL)
+            redis.setex(key, TTL, 1)
           end
 
           def agent

lib/travis/build/appliances/services.rb

@@ -10,7 +10,8 @@ class Services < Base
           'memcache'     => 'memcached',
           'neo4j-server' => 'neo4j',
           'rabbitmq'     => 'rabbitmq-server',
-          'redis'        => 'redis-server'
+          'redis'        => 'redis-server',
+          'cri-dockerd'  => 'cri_dockerd'
         }
 
         def apply
@@ -56,6 +57,16 @@ def apply_mongodb
           end
         end
 
+        def apply_cri_dockerd
+          sh.if '"$TRAVIS_OS_NAME" != linux' do
+            sh.echo "Addon cri-dockerd is not supported on #{data[:config][:os]}", ansi: :red
+          end
+          sh.else do
+            sh.raw bash('travis_setup_cri-dockerd'), echo: false, timing: false
+            sh.cmd "travis_setup_cri-dockerd", echo: true, timing: true
+          end
+        end
+
         def apply_mysql
           sh.raw <<~BASH
             travis_mysql_ping() {

lib/travis/build/appliances/set_docker_mtu_and_registry_mirrors.rb

@@ -6,6 +6,7 @@ module Appliances
       class SetDockerMtuAndRegistryMirrors < Base
 
         REGISTRY_URL = Travis::Build.config.registry_url.output_safe.freeze
+        MTU = Travis::Build.config.docker.mtu.to_i.freeze
 
         def apply?
           linux?
@@ -16,11 +17,11 @@ def apply
             sh.raw <<-EOF
 sudo test -f /etc/docker/daemon.json
 if [[ $? = 0 ]]; then
-  echo '[{"op":"add","path":"/mtu","value":1460}]' > mtu.jsonpatch
+  echo '[{"op":"add","path":"/mtu","value":#{MTU}}]' > mtu.jsonpatch
   sudo jsonpatch /etc/docker/daemon.json mtu.jsonpatch > daemon.json
   sudo mv daemon.json /etc/docker/daemon.json
 else
-  echo '{"mtu":1460}' | sudo tee /etc/docker/daemon.json > /dev/null
+  echo '{"mtu":#{MTU}}' | sudo tee /etc/docker/daemon.json > /dev/null
 fi
 
 if curl --connect-timeout 1 -fsSL -o /dev/null \

lib/travis/build/bash/travis_setup_cri-dockerd.bash

@@ -0,0 +1,26 @@
+travis_setup_cri-dockerd() {
+  local cri_containerd_cni_version='1.6.14'
+  local crictl_version='v1.24.2'
+  echo -e "${ANSI_YELLOW}cri-dockerd setup ${ANSI_CLEAR}"
+  sudo bash -c "
+    groupadd docker || true;
+    apt-get update && apt-get install socat eptables;
+    apt-get install conntrack containerd;
+    wget https://github.com/containerd/containerd/releases/download/v${cri_containerd_cni_version}/cri-containerd-cni-${cri_containerd_cni_version}-linux-amd64.tar.gz;
+    tar zxvf cri-containerd-cni-${cri_containerd_cni_version}-linux-amd64.tar.gz -C /;
+    rm -rf cri-containerd-cni-1.6.14-linux-amd64.tar.gz;
+    wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.0/cri-dockerd_0.3.0.3-0.ubuntu-bionic_amd64.deb;
+    dpkg -i cri-dockerd_0.3.0.3-0.ubuntu-bionic_amd64.deb;
+    wget https://raw.githubusercontent.com/Mirantis/cri-dockerd/v0.3.0/packaging/systemd/cri-docker.service
+    wget https://raw.githubusercontent.com/Mirantis/cri-dockerd/v0.3.0/packaging/systemd/cri-docker.socket;
+    mv cri-docker.socket cri-docker.service /etc/systemd/system/;
+    systemctl daemon-reload;
+    systemctl enable cri-docker.service;
+    systemctl enable --now cri-docker.socket;
+    wget https://github.com/kubernetes-sigs/cri-tools/releases/download/${crictl_version}/crictl-${crictl_version}-linux-amd64.tar.gz
+    tar zxvf crictl-${crictl_version}-linux-amd64.tar.gz -C /usr/bin;
+    rm -f crictl-${crictl_version}-linux-amd64.tar.gz;
+    echo runtime-endpoint: unix:///run/containerd/containerd.sock > /etc/crictl.yaml;
+    echo image-endpoint: unix:///run/containerd/containerd.sock >> /etc/crictl.yaml;
+"
+}

lib/travis/build/config.rb

@@ -108,6 +108,9 @@ def sc_data
         },
         maven_central_mirror: ENV.fetch('TRAVIS_MAVEN_CENTRAL_MIRROR', ''),
         registry_url: ENV.fetch('TRAVIS_BUILD_REGISTRY_URL', 'https://registry.travis-ci.com'),
+        docker: {
+          mtu: ENV.fetch('TRAVIS_BUILD_DOCKER_MTU', 1350)
+        },
         network: {
           wait_retries: Integer(ENV.fetch(
             'TRAVIS_BUILD_NETWORK_WAIT_RETRIES',

lib/travis/build/data.rb

@@ -100,7 +100,7 @@ def ssh_key?
       def ssh_key
         @ssh_key ||= if ssh_key = data[:ssh_key]
           SshKey.new(ssh_key[:value], ssh_key[:source], ssh_key[:encoded], ssh_key[:public_key])
-        elsif source_key = data[:config][:source_key]
+        elsif data[:config] && source_key = data[:config][:source_key]
           SshKey.new(source_key, nil, true, nil)
         end
       end
@@ -152,13 +152,17 @@ def source_https?
       def source_ssh?
         return false if prefer_https?
         ((repo_private? || force_private?) && !installation?) ||
-          (repo_private? && custom_ssh_key?)
+          ((repo_private? || enterprise?) && custom_ssh_key?)
       end
 
       def force_private?
         github? && !source_host&.include?('github.com')
       end
 
+      def enterprise?
+        ENV['TRAVIS_ENTERPRISE'] == 'true' || nil
+      end
+
       def github?
         repository[:vcs_type] == 'GithubRepository'
       end

lib/travis/build/script.rb

@@ -442,6 +442,7 @@ def error_message_ary(exception, event)
         end
 
         def show_compile_error_msg(exception, event)
+          puts "Exception Occurred when trying to compile #{exception.class}. Message: #{exception.message}. Backtrace:  \n #{exception.backtrace.join("\n")}"
           @sh = Shell::Builder.new
           error_message_ary(exception, event).each { |line| sh.raw "echo -e \"\033[31;1m#{line}\033[0m\"" }
           sh.raw "exit 2"

lib/travis/build/script/node_js.rb

@@ -31,7 +31,6 @@ def setup
           sh.newline
 
           npm_disable_prefix
-          npm_disable_spinner
           npm_disable_progress
           npm_disable_strict_ssl unless npm_strict_ssl?
           install_yarn_when_locked

lib/travis/services/vault/keys/kv1.rb

@@ -4,8 +4,8 @@ module Travis
   module Vault
     class Keys
       class KV1
-        def self.resolve(path, vault)
-          response = RestClient.get("#{vault[:api_url]}/v1/secret/#{path}", 'X-Vault-Token': vault[:token])
+        def self.resolve(namespace, mount, path, vault)
+          response = RestClient.get("#{vault[:api_url]}/v1/#{mount}/#{path}", 'X-Vault-Token': vault[:token], 'X-Vault-Namespace': namespace ? namespace : "")
           JSON.parse(response.body)['data'] if response.code == 200
         rescue RestClient::ExceptionWithResponse, SocketError
           nil

lib/travis/services/vault/keys/kv2.rb

@@ -4,8 +4,8 @@ module Travis
   module Vault
     class Keys
       class KV2
-        def self.resolve(path, vault)
-          response = RestClient.get("#{vault[:api_url]}/v1/secret/data/#{path}", 'X-Vault-Token': vault[:token])
+        def self.resolve(namespace, mount, path, vault)
+          response = RestClient.get("#{vault[:api_url]}/v1/#{mount}/data/#{path}", 'X-Vault-Token': vault[:token], 'X-Vault-Namespace': namespace ? namespace : "")
           JSON.parse(response.body).dig('data', 'data') if response.code == 200
         rescue RestClient::ExceptionWithResponse, SocketError
           nil

lib/travis/services/vault/keys/resolver.rb

@@ -22,10 +22,18 @@ def initialize(paths, version, appliance)
         def call
           return if paths.blank?
 
+          namespace = nil
           vault_secrets = []
 
+          if appliance.vault.is_a?(Hash)
+            secrets = appliance.vault[:secrets]
+            namespace = secrets[:namespace].find { |el| el.is_a?(Hash) && el&.dig(:name) }&.dig(:name) if secrets&.include?(:namespace)
+          end
           paths.each do |path|
-            secret_data = Keys.const_get(version.upcase).resolve(path, vault)
+            parts = path.split('/',2)
+            mount = parts&.first
+            path = parts&.last
+            secret_data = Keys.const_get(version.upcase).resolve(namespace, mount, path, vault)
             if secret_data.present?
               secret_name = path.split('/').last
               secret_data.each do |key, value|

public/version-aliases/ghc.json

@@ -166,9 +166,9 @@
   "9.0.1-alpha1": "9.0.1-alpha1",
   "9.0": "9.0.2",
   "9.0.1": "9.0.1",
-  "9": "9.8.2",
-  "9.x": "9.8.2",
-  "9.x.x": "9.8.2",
+  "9": "9.10.1",
+  "9.x": "9.10.1",
+  "9.x.x": "9.10.1",
   "9.0.x": "9.0.2",
   "9.0.2": "9.0.2",
   "9.2.1-alpha1": "9.2.1-alpha1",
@@ -197,8 +197,8 @@
   "9.4.7": "9.4.7",
   "9.4.8": "9.4.8",
   "9.6.0.20230111": "9.6.0.20230111",
-  "9.6.x": "9.6.4",
-  "9.6": "9.6.4",
+  "9.6.x": "9.6.5",
+  "9.6": "9.6.5",
   "9.6.0.20230128": "9.6.0.20230128",
   "9.6.0.20230210": "9.6.0.20230210",
   "9.6.0.20230302": "9.6.0.20230302",
@@ -209,6 +209,7 @@
   "9.6.2": "9.6.2",
   "9.6.3": "9.6.3",
   "9.6.4": "9.6.4",
+  "9.6.5": "9.6.5",
   "9.8.0.20230727": "9.8.0.20230727",
   "9.8.x": "9.8.2",
   "9.8": "9.8.2",
@@ -221,5 +222,15 @@
   "9.8.1-alpha3": "9.8.1-alpha3",
   "9.8.1-alpha4": "9.8.1-alpha4",
   "9.8.1": "9.8.1",
-  "9.8.2": "9.8.2"
+  "9.8.2": "9.8.2",
+  "9.10.0.20240313": "9.10.0.20240313",
+  "9.10.x": "9.10.1",
+  "9.10": "9.10.1",
+  "9.10.0.20240328": "9.10.0.20240328",
+  "9.10.0.20240413": "9.10.0.20240413",
+  "9.10.0.20240426": "9.10.0.20240426",
+  "9.10.1-alpha1": "9.10.1-alpha1",
+  "9.10.1-alpha2": "9.10.1-alpha2",
+  "9.10.1-alpha3": "9.10.1-alpha3",
+  "9.10.1": "9.10.1"
 }

spec/build/data_spec.rb

@@ -124,5 +124,37 @@
       it { expect(data.source_ssh?).to be nil }
       it { expect(data.token).to eq 'access_token' }
     end
+
+    describe 'source_ssh is true for public enterprise repository' do
+      let(:config) { {oauth_token: 'access_token', prefer_https: false, repository: { installation_id: 1, private: false, vcs_id: 123} } }
+      let(:data) { Travis::Build::Data.new(config) }
+
+      before {
+        Travis::GithubApps.any_instance.stubs(:access_token).returns 'access_token'
+        ENV['TRAVIS_ENTERPRISE'] = 'true'
+      }
+      after {
+        ENV['TRAVIS_ENTERPRISE'] = nil
+      }
+      it { expect(data.installation?).to be true }
+      it { expect(data.source_ssh?).to be false }
+      it { expect(data.token).to eq 'access_token' }
+    end
+
+    describe 'source_ssh is true for public enterprise repository if custom key is present' do
+      let(:config) { {oauth_token: 'access_token', prefer_https: false, repository: { installation_id: 1, private: false, vcs_id: 123}, ssh_key: { value: TEST_PRIVATE_KEY, source: 'repository_settings' } } }
+      let(:data) { Travis::Build::Data.new(config) }
+
+      before {
+        Travis::GithubApps.any_instance.stubs(:access_token).returns 'access_token'
+        ENV['TRAVIS_ENTERPRISE'] = 'true'
+      }
+      after {
+        ENV['TRAVIS_ENTERPRISE'] = nil
+      }
+      it { expect(data.installation?).to be true }
+      it { expect(data.source_ssh?).to be true }
+      it { expect(data.token).to eq 'access_token' }
+    end
   end
 end

spec/build/script/node_js_spec.rb

@@ -111,10 +111,6 @@
     should include_sexp [:cmd, 'npm --version', echo: true]
   end
 
-  it 'disables the npm spinner' do
-    should include_sexp [:cmd, 'npm config set spin false || true', assert: true]
-  end
-
   describe 'if package.json exists' do
     let(:sexp) { sexp_find(subject, [:if, '-f package.json'], [:then]) }
 

spec/build/services/vault/keys/kv1_spec.rb

@@ -2,7 +2,7 @@
 
 describe Travis::Vault::Keys::KV1 do
   describe '.resolve' do
-    subject { described_class.resolve(path, vault) }
+    subject { described_class.resolve('', 'secret', path, vault) }
 
     let(:vault) do
       {

spec/build/services/vault/keys/kv2_spec.rb

@@ -2,7 +2,7 @@
 
 describe Travis::Vault::Keys::KV2 do
   describe '.resolve' do
-    subject { described_class.resolve(path, vault) }
+    subject { described_class.resolve('', 'secret', path, vault) }
 
     let(:vault) do
       {