Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Reporting] code cleanup for reporting browser build/install/setup utilities #98799

Merged
merged 12 commits into from
Apr 30, 2021
13 changes: 9 additions & 4 deletions src/dev/chromium_version.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,13 @@ async function getPuppeteerRelease(log: ToolingLog): Promise<PuppeteerRelease> {
'Could not get the Puppeteer version! Check node_modules/puppteer/package.json'
);
}
log.info(`Kibana is using Puppeteer ${version} (${forkCompatibilityMap[version]})`);
return forkCompatibilityMap[version];
let puppeteerRelease = forkCompatibilityMap[version];
if (puppeteerRelease == null) {
puppeteerRelease = version;
}
tsullivan marked this conversation as resolved.
Show resolved Hide resolved

log.info(`Kibana is using Puppeteer ${version} (${puppeteerRelease})`);
return puppeteerRelease;
}

async function getChromiumRevision(
Expand Down Expand Up @@ -129,8 +134,8 @@ run(
description: chalk`
Display the Chromium git commit that correlates to a given Puppeteer release.

- node x-pack/dev-tools/chromium_version 5.5.0 {dim # gets the Chromium commit for Puppeteer v5.5.0}
- node x-pack/dev-tools/chromium_version {dim # gets the Chromium commit for the Kibana dependency version of Puppeteer}
- node scripts/chromium_version 5.5.0 {dim # gets the Chromium commit for Puppeteer v5.5.0}
- node scripts/chromium_version {dim # gets the Chromium commit for the Kibana dependency version of Puppeteer}

You can use https://omahaproxy.appspot.com/ to look up the Chromium release that first shipped with that commit.
`,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,14 @@ const log = new ToolingLog({
});

describe(`enumeratePatterns`, () => {
it(`should resolve x-pack/plugins/reporting/server/browsers/extract/unzip.js to kibana-reporting`, () => {
it(`should resolve x-pack/plugins/reporting/server/browsers/extract/unzip.ts to kibana-reporting`, () => {
const actual = enumeratePatterns(REPO_ROOT)(log)(
new Map([['x-pack/plugins/reporting', ['kibana-reporting']]])
);

expect(
actual[0].includes(
'x-pack/plugins/reporting/server/browsers/extract/unzip.js kibana-reporting'
'x-pack/plugins/reporting/server/browsers/extract/unzip.ts kibana-reporting'
)
).toBe(true);
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -148,10 +148,10 @@ x-pack/plugins/reporting/server/browsers/download/download.ts kibana-reporting
x-pack/plugins/reporting/server/browsers/download/ensure_downloaded.ts kibana-reporting
x-pack/plugins/reporting/server/browsers/download/index.ts kibana-reporting
x-pack/plugins/reporting/server/browsers/download/util.ts kibana-reporting
x-pack/plugins/reporting/server/browsers/extract/extract.js kibana-reporting
x-pack/plugins/reporting/server/browsers/extract/extract_error.js kibana-reporting
x-pack/plugins/reporting/server/browsers/extract/index.js kibana-reporting
x-pack/plugins/reporting/server/browsers/extract/unzip.js kibana-reporting
x-pack/plugins/reporting/server/browsers/extract/extract.ts kibana-reporting
x-pack/plugins/reporting/server/browsers/extract/extract_error.ts kibana-reporting
x-pack/plugins/reporting/server/browsers/extract/index.ts kibana-reporting
x-pack/plugins/reporting/server/browsers/extract/unzip.ts kibana-reporting
x-pack/plugins/reporting/server/browsers/index.ts kibana-reporting
x-pack/plugins/reporting/server/browsers/install.ts kibana-reporting
x-pack/plugins/reporting/server/browsers/network_policy.test.ts kibana-reporting
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,27 +32,27 @@ describe(`Transform fns`, () => {
it(`should remove the jenkins workspace path`, () => {
const obj = {
staticSiteUrl:
'/var/lib/jenkins/workspace/elastic+kibana+code-coverage/kibana/x-pack/plugins/reporting/server/browsers/extract/unzip.js',
'/var/lib/jenkins/workspace/elastic+kibana+code-coverage/kibana/x-pack/plugins/reporting/server/browsers/extract/unzip.ts',
COVERAGE_INGESTION_KIBANA_ROOT:
'/var/lib/jenkins/workspace/elastic+kibana+code-coverage/kibana',
};
expect(coveredFilePath(obj)).toHaveProperty(
'coveredFilePath',
'x-pack/plugins/reporting/server/browsers/extract/unzip.js'
'x-pack/plugins/reporting/server/browsers/extract/unzip.ts'
);
});
});
describe(`in the qa research job`, () => {
it(`should remove the jenkins workspace path`, () => {
const obj = {
staticSiteUrl:
'/var/lib/jenkins/workspace/elastic+kibana+qa-research/kibana/x-pack/plugins/reporting/server/browsers/extract/unzip.js',
'/var/lib/jenkins/workspace/elastic+kibana+qa-research/kibana/x-pack/plugins/reporting/server/browsers/extract/unzip.ts',
COVERAGE_INGESTION_KIBANA_ROOT:
'/var/lib/jenkins/workspace/elastic+kibana+qa-research/kibana',
};
expect(coveredFilePath(obj)).toHaveProperty(
'coveredFilePath',
'x-pack/plugins/reporting/server/browsers/extract/unzip.js'
'x-pack/plugins/reporting/server/browsers/extract/unzip.ts'
);
});
});
Expand Down Expand Up @@ -82,7 +82,7 @@ describe(`Transform fns`, () => {
describe(`teamAssignment`, () => {
const teamAssignmentsPathMOCK =
'src/dev/code_coverage/ingest_coverage/__tests__/mocks/team_assign_mock.txt';
const coveredFilePath = 'x-pack/plugins/reporting/server/browsers/extract/unzip.js';
const coveredFilePath = 'x-pack/plugins/reporting/server/browsers/extract/unzip.ts';
const obj = { coveredFilePath };
const log = new ToolingLog({
level: 'info',
Expand Down
2 changes: 1 addition & 1 deletion x-pack/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
/target
/test/functional/failure_debug
/test/functional/screenshots
/test/functional/apps/reporting/reports/session
/test/functional/apps/**/reports/session
/test/reporting/configs/failure_debug/
/plugins/reporting/.chromium/
/plugins/reporting/chromium/
Expand Down
26 changes: 17 additions & 9 deletions x-pack/build_chromium/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ which is where we have two machines provisioned for the Linux and Windows
builds. Mac builds can be achieved locally, and are a great place to start to
gain familiarity.

**NOTE:** Linux builds should be done in Ubuntu on x86 architecture. ARM builds
are created in x86. CentOS is not supported for building Chromium.
**NOTE:** Linux builds should be done in Ubuntu on x64 architecture. ARM builds
are created in x64 using cross-compiling. CentOS is not supported for building Chromium.

1. Login to our GCP instance [here using your okta credentials](https://console.cloud.google.com/).
2. Click the "Compute Engine" tab.
Expand All @@ -27,25 +27,32 @@ are created in x86. CentOS is not supported for building Chromium.
- python2 (`python` must link to `python2`)
- lsb_release
- tmux is recommended in case your ssh session is interrupted
6. Copy the entire `build_chromium` directory into a GCP storage bucket, so you can copy the scripts into the instance and run them.
- "Cloud API access scopes": must have **read / write** scope for the Storage API
6. Copy the entire `build_chromium` directory from the `headless_shell_staging` bucket. To do this, use `gsutil rsync`:
```sh
# This shows a preview of what would change by synchronizing the source scripts with the destination GCS bucket.
# Remove the `-n` flag to enact the changes
gsutil -m rsync -n -r x-pack/build_chromium gs://headless_shell_staging/build_chromium
```

## Build Script Usage

```
These commands show how to set up an environment to build:
```sh
# Allow our scripts to use depot_tools commands
export PATH=$HOME/chromium/depot_tools:$PATH

# Create a dedicated working directory for this directory of Python scripts.
mkdir ~/chromium && cd ~/chromium

# Copy the scripts from the Kibana repo to use them conveniently in the working directory
gsutil cp -r gs://my-bucket/build_chromium .
# Copy the scripts from the Kibana team's GCS bucket
gsutil cp -r gs://headless_shell_staging/build_chromium .

# Install the OS packages, configure the environment, download the chromium source (25GB)
python ./build_chromium/init.sh [arch_name]
python ./build_chromium/init.py [arch_name]

# Run the build script with the path to the chromium src directory, the git commit hash
python ./build_chromium/build.py <commit_id> x86
python ./build_chromium/build.py <commit_id> x64

# OR You can build for ARM
python ./build_chromium/build.py <commit_id> arm64
Expand Down Expand Up @@ -107,7 +114,7 @@ use the Kibana `build.py` script (in this directory).

It's recommended that you create a working directory for the chromium source
code and all the build tools, and run the commands from there:
```
```sh
mkdir ~/chromium && cd ~/chromium
cp -r ~/path/to/kibana/x-pack/build_chromium .
python ./build_chromium/init.sh [arch_name]
Expand Down Expand Up @@ -216,6 +223,7 @@ In the case of Windows, you can use IE to open `http://localhost:9221` and see i

The following links provide helpful context about how the Chromium build works, and its prerequisites:

- Tools for Chromium version information: https://omahaproxy.appspot.com/
- https://www.chromium.org/developers/how-tos/get-the-code/working-with-release-branches
- https://chromium.googlesource.com/chromium/src/+/HEAD/docs/windows_build_instructions.md
- https://chromium.googlesource.com/chromium/src/+/HEAD/docs/mac_build_instructions.md
Expand Down
49 changes: 29 additions & 20 deletions x-pack/build_chromium/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,18 @@
from build_util import (
runcmd,
runcmdsilent,
mkdir,
md5_file,
configure_environment,
)

# This file builds Chromium headless on Windows, Mac, and Linux.

# Verify that we have an argument, and if not print instructions
if (len(sys.argv) < 2):
print('Usage:')
print('python build.py {chromium_version} [arch_name]')
print('python build.py {chromium_version} {arch_name}')
print('Example:')
print('python build.py 68.0.3440.106')
print('python build.py 4747cc23ae334a57a35ed3c8e6adcdbc8a50d479')
print('python build.py 4747cc23ae334a57a35ed3c8e6adcdbc8a50d479 arm64 # build for ARM architecture')
print('python build.py 4747cc23ae334a57a35ed3c8e6adcdbc8a50d479 x64')
print('python build.py 4747cc23ae334a57a35ed3c8e6adcdbc8a50d479 arm64 # cross-compile for ARM architecture')
print
sys.exit(1)

Expand Down Expand Up @@ -57,25 +54,34 @@
print('Creating a new branch for tracking the source version')
runcmd('git checkout -b build-' + base_version + ' ' + source_version)

# configure environment: environment path
depot_tools_path = os.path.join(build_path, 'depot_tools')
path_value = depot_tools_path + os.pathsep + os.environ['PATH']
print('Updating PATH for depot_tools: ' + path_value)
os.environ['PATH'] = path_value
full_path = depot_tools_path + os.pathsep + os.environ['PATH']
print('Updating PATH for depot_tools: ' + full_path)
os.environ['PATH'] = full_path

# configure environment: build dependencies
if platform.system() == 'Linux':
if arch_name:
print('Running sysroot install script...')
runcmd(src_path + '/build/linux/sysroot_scripts/install-sysroot.py --arch=' + arch_name)
print('Running install-build-deps...')
runcmd(src_path + '/build/install-build-deps.sh')


print('Updating all modules')
runcmd('gclient sync')
runcmd('gclient sync -D')

# Copy build args/{Linux | Darwin | Windows}.gn from the root of our directory to out/headless/args.gn,
argsgn_destination = path.abspath('out/headless/args.gn')
print('Generating platform-specific args')
mkdir('out/headless')
print(' > cp ' + argsgn_file + ' ' + argsgn_destination)
shutil.copyfile(argsgn_file, argsgn_destination)
print('Setting up build directory')
runcmd('rm -rf out/headless')
runcmd('mkdir out/headless')

# Copy build args/{Linux | Darwin | Windows}.gn from the root of our directory to out/headless/args.gn,
# add the target_cpu for cross-compilation
print('Adding target_cpu to args')

f = open('out/headless/args.gn', 'a')
f.write('\rtarget_cpu = "' + arch_name + '"\r')
f.close()
argsgn_file_out = path.abspath('out/headless/args.gn')
runcmd('cp ' + argsgn_file + ' ' + argsgn_file_out)
runcmd('echo \'target_cpu="' + arch_name + '"\' >> ' + argsgn_file_out)
Copy link
Member Author

@tsullivan tsullivan Apr 29, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the f.write code was not working most of the time: seemed to be a race condition where the previous line of code would complete later than expected


runcmd('gn gen out/headless')

Expand Down Expand Up @@ -136,3 +142,6 @@ def archive_file(name):
print('Creating ' + path.join(src_path, md5_filename))
with open (md5_filename, 'w') as f:
f.write(md5_file(zip_filename))

runcmd('gsutil cp ' + path.join(src_path, zip_filename) + ' gs://headless_shell_staging')
runcmd('gsutil cp ' + path.join(src_path, md5_filename) + ' gs://headless_shell_staging')
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cc @spalger : this adds an auto-upload to a staging bucket as we talked about.

I think this suffices the problem we had here: #98688 (comment)

This is without adding a dry-run option to the script, as discussed. LMK if you think it needs the dry-run option. (It would be a pain since a run takes hours.)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I think this is the best approach, then staging assets can just be gsutil mvd to the headless_shell bucket when they're verified.

Might be nice to use an environment variable to switch out the URLs to actually download from staging so we can test things before promoting them.

16 changes: 0 additions & 16 deletions x-pack/build_chromium/build_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,19 +27,3 @@ def md5_file(filename):
for chunk in iter(lambda: f.read(128 * md5.block_size), b''):
md5.update(chunk)
return md5.hexdigest()

def configure_environment(arch_name, build_path, src_path):
"""Runs install scripts for deps, and configures temporary environment variables required by Chromium's build"""

if platform.system() == 'Linux':
if arch_name:
print('Running sysroot install script...')
sysroot_cmd = src_path + '/build/linux/sysroot_scripts/install-sysroot.py --arch=' + arch_name
runcmd(sysroot_cmd)
print('Running install-build-deps...')
runcmd(src_path + '/build/install-build-deps.sh')

depot_tools_path = os.path.join(build_path, 'depot_tools')
full_path = depot_tools_path + os.pathsep + os.environ['PATH']
print('Updating PATH for depot_tools: ' + full_path)
os.environ['PATH'] = full_path
8 changes: 5 additions & 3 deletions x-pack/build_chromium/darwin/args.gn
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,15 @@ use_alsa = false
use_cups = false
use_dbus = false
use_gio = false
# Please, consult @elastic/kibana-security before changing/removing this option.
use_kerberos = false
use_libpci = false
use_pulseaudio = false
use_udev = false

is_debug = false
symbol_level = 0
is_component_build = false
remove_webcore_debug_symbols = true
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

remove_webcore_debug_symbols caused warning logs during compilation: it is unused


# Please, consult @elastic/kibana-security before changing/removing this option.
use_kerberos = false
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

moved these lines down to match the order of lines with ../windows/args.gn


# target_cpu is appended before build: "x64" or "arm64"
5 changes: 1 addition & 4 deletions x-pack/build_chromium/init.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import os, platform, sys
from os import path
from build_util import runcmd, mkdir, md5_file, configure_environment
from build_util import runcmd, mkdir

# This is a cross-platform initialization script which should only be run
# once per environment, and isn't intended to be run directly. You should
Expand Down Expand Up @@ -44,6 +44,3 @@
runcmd('fetch chromium --nohooks=1 --no-history=1')
else:
print('Directory exists: ' + chromium_dir + '. Skipping chromium fetch.')

# This depends on having the chromium/src directory with the complete checkout
configure_environment(arch_name, build_path, src_path)
3 changes: 2 additions & 1 deletion x-pack/build_chromium/linux/args.gn
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@ import("//build/args/headless.gn")
is_debug = false
symbol_level = 0
is_component_build = false
remove_webcore_debug_symbols = true
enable_nacl = false
# Please, consult @elastic/kibana-security before changing/removing this option.
use_kerberos = false

# target_cpu is appended before build: "x64" or "arm64"
8 changes: 5 additions & 3 deletions x-pack/build_chromium/windows/args.gn
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,12 @@ use_gio = false
use_libpci = false
use_pulseaudio = false
use_udev = false
# Please, consult @elastic/kibana-security before changing/removing this option.
use_kerberos = false

is_debug = false
symbol_level = 0
is_component_build = false
remove_webcore_debug_symbols = true

# Please, consult @elastic/kibana-security before changing/removing this option.
use_kerberos = false

# target_cpu is appended before build: "x64" or "arm64"
Loading