Skip to content

Commit

Permalink
refactor: combine package_resolution and package_index
Browse files Browse the repository at this point in the history
Previously we had:
```starlark
    pkgindex = package_index.new(rctx, sources = sources, archs = manifest["archs"])
    pkgresolution = package_resolution.new(index = pkgindex)
```
And none of the code of package_resolution was used anywhere but in
resolve.bzl and after initializing the `pkgindex`.

Also, it makes sense since we are building the index from the manifest
and once we have the index we use it to resolve the packages and
populate the lock.
  • Loading branch information
jjmaestro committed Sep 19, 2024
1 parent e274cb8 commit 95da3ce
Show file tree
Hide file tree
Showing 7 changed files with 264 additions and 284 deletions.
7 changes: 0 additions & 7 deletions apt/private/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -43,12 +43,6 @@ bzl_library(
name = "package_index",
srcs = ["package_index.bzl"],
visibility = ["//apt:__subpackages__"],
)

bzl_library(
name = "package_resolution",
srcs = ["package_resolution.bzl"],
visibility = ["//apt:__subpackages__"],
deps = [":version"],
)

Expand All @@ -59,7 +53,6 @@ bzl_library(
deps = [
":lockfile",
":package_index",
":package_resolution",
"@aspect_bazel_lib//lib:repo_utils",
],
)
Expand Down
158 changes: 157 additions & 1 deletion apt/private/package_index.bzl
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
"package index"

load(":version.bzl", version_lib = "version")

def _fetch_package_index(rctx, url, dist, comp, arch):
# See https://linux.die.net/man/1/xz and https://linux.die.net/man/1/gzip
# --keep -> keep the original file (Bazel might be still committing the output to the cache)
Expand Down Expand Up @@ -123,7 +125,7 @@ def _package_get(packages, arch, name, version = None):

return versions.get(version, None)

def _new(rctx, sources, archs):
def _index(rctx, sources, archs):
packages = {}

for arch in archs:
Expand All @@ -148,11 +150,165 @@ def _new(rctx, sources, archs):
package_get = lambda arch, name, version = None: _package_get(packages, arch, name, version),
)

# "package resolution"

def _parse_dep(raw):
raw = raw.strip() # remove leading & trailing whitespace
name = None
version = None
archs = None

sqb_start_i = raw.find("[")
if sqb_start_i != -1:
sqb_end_i = raw.find("]")
if sqb_end_i == -1:
fail('invalid version string %s expected a closing brackets "]"' % raw)
archs = raw[sqb_start_i + 1:sqb_end_i].strip().split(" ")
raw = raw[:sqb_start_i] + raw[sqb_end_i + 1:]

paren_start_i = raw.find("(")
if paren_start_i != -1:
paren_end_i = raw.find(")")
if paren_end_i == -1:
fail('invalid version string %s expected a closing paren ")"' % raw)
name = raw[:paren_start_i].strip()
version_and_constraint = raw[paren_start_i + 1:paren_end_i].strip()
version = version_lib.parse_version_and_constraint(version_and_constraint)
raw = raw[:paren_start_i] + raw[paren_end_i + 1:]

# Depends: python3:any
# is equivalent to
# Depends: python3 [any]
colon_i = raw.find(":")
if colon_i != -1:
arch_after_colon = raw[colon_i + 1:]
raw = raw[:colon_i]
archs = [arch_after_colon.strip()]

name = raw.strip()
return {"name": name, "version": version, "arch": archs}

def _parse_depends(depends_raw):
depends = []
for dep in depends_raw.split(","):
if dep.find("|") != -1:
depends.append([
_parse_dep(adep)
for adep in dep.split("|")
])
else:
depends.append(_parse_dep(dep))

return depends

def _resolve_package(index, arch, name, version):
# Get available versions of the package
versions = index.package_get(arch, name)

# Order packages by highest to lowest
versions = version_lib.sort(versions, reverse = True)
package = None
if version:
for va in versions:
op, vb = version
if version_lib.compare(va, op, vb):
package = index.package_get(arch, name, va)

# Since versions are ordered by hight to low, the first
# satisfied version will be the highest version and
# rules_distroless ignores Priority field so it's safe.
# TODO: rethink this `break` with issue #34
break
elif len(versions) > 0:
# First element in the versions list is the latest version.
version = versions[0]
package = index.package_get(arch, name, version)
return package

def _resolve_all(index, arch, name, version, include_transitive):
root_package = None
already_recursed = {}
unmet_dependencies = []
dependencies = []
has_optional_deps = False
iteration_max = 2147483646

stack = [(name, version)]

for i in range(0, iteration_max + 1):
if not len(stack):
break
if i == iteration_max:
fail("resolve_dependencies exhausted the iteration")
(name, version) = stack.pop()

package = _resolve_package(index, arch, name, version)

if not package:
key = "%s~~%s" % (name, version[1] if version else "")
unmet_dependencies.append((name, version))
continue

if i == 0:
# Set the root package
root_package = package

key = "%s~~%s" % (package["Package"], package["Version"])

# If we encountered package before in the transitive closure, skip it
if key in already_recursed:
continue

if i != 0:
# Add it to the dependencies
already_recursed[key] = True
dependencies.append(package)

deps = []

# Extend the lookup with all the items in the dependency closure
if "Pre-Depends" in package and include_transitive:
deps.extend(_parse_depends(package["Pre-Depends"]))

# Extend the lookup with all the items in the dependency closure
if "Depends" in package and include_transitive:
deps.extend(_parse_depends(package["Depends"]))

for dep in deps:
if type(dep) == "list":
# TODO: optional dependencies
has_optional_deps = True
continue

# TODO: arch
stack.append((dep["name"], dep["version"]))

if has_optional_deps:
msg = "Warning: package '{}/{}' (or one of its dependencies) "
msg += "has optional dependencies that are not supported yet: #27"
print(msg.format(root_package["Package"], arch))

if unmet_dependencies:
msg = "Warning: the following packages have unmet dependencies: {}"
print(msg.format(",".join([up[0] for up in unmet_dependencies])))

return root_package, dependencies

def _new(rctx, sources, archs):
index = _index(rctx, sources, archs)

return struct(
resolve_all = lambda **kwargs: _resolve_all(index, **kwargs),
resolve_package = lambda **kwargs: _resolve_package(index, **kwargs),
)

package_index = struct(
new = _new,
parse_depends = _parse_depends,
# NOTE: these are exposed here for testing purposes, DO NOT USE OTHERWISE
_fetch_package_index = _fetch_package_index,
_parse_package_index = _parse_package_index,
_package_set = _package_set,
_package_get = _package_get,
_index = _index,
)
156 changes: 0 additions & 156 deletions apt/private/package_resolution.bzl

This file was deleted.

6 changes: 2 additions & 4 deletions apt/private/resolve.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
load("@aspect_bazel_lib//lib:repo_utils.bzl", "repo_utils")
load(":lockfile.bzl", "lockfile")
load(":package_index.bzl", "package_index")
load(":package_resolution.bzl", "package_resolution")

def _parse_manifest(rctx, yq_toolchain_prefix, manifest):
is_windows = repo_utils.is_windows(rctx)
Expand Down Expand Up @@ -49,7 +48,6 @@ def internal_resolve(rctx, yq_toolchain_prefix, manifest, include_transitive):
))

pkgindex = package_index.new(rctx, sources = sources, archs = manifest["archs"])
pkgresolution = package_resolution.new(index = pkgindex)
lockf = lockfile.empty(rctx)

for arch in manifest["archs"]:
Expand All @@ -59,10 +57,10 @@ def internal_resolve(rctx, yq_toolchain_prefix, manifest, include_transitive):
fail("Duplicate package, {}. Please remove it from your manifest".format(dep_constraint))
dep_constraint_set[dep_constraint] = True

constraint = package_resolution.parse_depends(dep_constraint).pop()
constraint = package_index.parse_depends(dep_constraint).pop()

rctx.report_progress("Resolving %s" % dep_constraint)
package, dependencies = pkgresolution.resolve_all(
package, dependencies = pkgindex.resolve_all(
arch = arch,
name = constraint["name"],
version = constraint["version"],
Expand Down
3 changes: 0 additions & 3 deletions apt/tests/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
load(":package_index_test.bzl", "package_index_tests")
load(":package_resolution_test.bzl", "package_resolution_tests")
load(":version_test.bzl", "version_tests")

package_index_tests()

package_resolution_tests()

version_tests()
Loading

0 comments on commit 95da3ce

Please sign in to comment.