Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Support passing extra args to pip install #655

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

feat: Support passing extra args to pip install #655

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
fc58359
feat: Support passing extra args to pip install
LironEr Jan 17, 2025
f5f4510
run precommit
LironEr Jan 17, 2025
309eb12
ruff fix
LironEr Jan 17, 2025
e0e07b9
Revert "run precommit"
LironEr Jan 17, 2025
b827f60
Merge branch 'master' into feat/pip-args
LironEr Jan 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,7 @@ source_path = [
- `:zip [source] [destination]` is a special command which creates content of current working directory (first argument) and places it inside of path (second argument).
- `pip_requirements` - Controls whether to execute `pip install`. Set to `false` to disable this feature, `true` to run `pip install` with `requirements.txt` found in `path`. Or set to another filename which you want to use instead. When `source_path` is passed as a string containing a path (and not a list of maps), and `requirements.txt` is present, `pip install` is automatically executed.
- `pip_tmp_dir` - Set the base directory to make the temporary directory for pip installs. Can be useful for Docker in Docker builds.
- `pip_install_extra_args` - A list of additional pip arguments to add to the pip install command
- `poetry_install` - Controls whether to execute `poetry export` and `pip install`. Set to `false` to disable this feature, `true` to run `poetry export` with `pyproject.toml` and `poetry.lock` found in `path`. When `source_path` is passed as a string containing a path (and not a list of maps), and `pyproject.toml` with a build system `poetry` is present, `poetry export` and `pip install` are automatically executed.
- `poetry_export_extra_args` - A list of additional poetry arguments to add to the poetry export command
- `npm_requirements` - Controls whether to execute `npm install`. Set to `false` to disable this feature, `true` to run `npm install` with `package.json` found in `path`. Or set to another filename which you want to use instead.
Expand Down
72 changes: 58 additions & 14 deletions package.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -687,7 +687,9 @@ def step(*x):
def hash(path):
source_paths.append(path)

def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None):
def pip_requirements_step(
path, prefix=None, required=False, tmp_dir=None, pip_install_extra_args=[]
):
command = runtime
requirements = path
if os.path.isdir(path):
Expand All @@ -703,11 +705,23 @@ def pip_requirements_step(path, prefix=None, required=False, tmp_dir=None):
"available in system PATH".format(command)
)

step("pip", runtime, requirements, prefix, tmp_dir)
step(
"pip",
runtime,
requirements,
prefix,
tmp_dir,
pip_install_extra_args,
)
hash(requirements)

def poetry_install_step(
path, poetry_export_extra_args=[], prefix=None, required=False, tmp_dir=None
path,
poetry_export_extra_args=[],
prefix=None,
required=False,
tmp_dir=None,
pip_install_extra_args=[],
):
pyproject_file = path
if os.path.isdir(path):
Expand All @@ -718,7 +732,15 @@ def poetry_install_step(
"poetry configuration not found: {}".format(pyproject_file)
)
else:
step("poetry", runtime, path, poetry_export_extra_args, prefix, tmp_dir)
step(
"poetry",
runtime,
path,
poetry_export_extra_args,
prefix,
tmp_dir,
pip_install_extra_args,
)
hash(pyproject_file)
pyproject_path = os.path.dirname(pyproject_file)
poetry_lock_file = os.path.join(pyproject_path, "poetry.lock")
Expand Down Expand Up @@ -819,6 +841,7 @@ def commands_step(path, commands):
else:
prefix = claim.get("prefix_in_zip")
pip_requirements = claim.get("pip_requirements")
pip_install_extra_args = claim.get("pip_install_extra_args", [])
poetry_install = claim.get("poetry_install")
poetry_export_extra_args = claim.get("poetry_export_extra_args", [])
npm_requirements = claim.get(
Expand All @@ -833,13 +856,15 @@ def commands_step(path, commands):
prefix,
required=True,
tmp_dir=claim.get("pip_tmp_dir"),
pip_install_extra_args=pip_install_extra_args,
)
else:
pip_requirements_step(
pip_requirements,
prefix,
required=True,
tmp_dir=claim.get("pip_tmp_dir"),
pip_install_extra_args=pip_install_extra_args,
)

if poetry_install and runtime.startswith("python"):
Expand All @@ -850,6 +875,7 @@ def commands_step(path, commands):
poetry_export_extra_args=poetry_export_extra_args,
required=True,
tmp_dir=claim.get("poetry_tmp_dir"),
pip_install_extra_args=pip_install_extra_args,
)

if npm_requirements and runtime.startswith("nodejs"):
Expand Down Expand Up @@ -937,9 +963,15 @@ def execute(self, build_plan, zip_stream, query):
else:
zs.write_file(source_path, prefix=prefix, timestamp=ts)
elif cmd == "pip":
runtime, pip_requirements, prefix, tmp_dir = action[1:]
(
runtime,
pip_requirements,
prefix,
tmp_dir,
pip_install_extra_args,
) = action[1:]
with install_pip_requirements(
query, pip_requirements, tmp_dir
query, pip_requirements, tmp_dir, pip_install_extra_args
) as rd:
if rd:
if pf:
Expand All @@ -950,12 +982,21 @@ def execute(self, build_plan, zip_stream, query):
# XXX: timestamp=0 - what actually do with it?
zs.write_dirs(rd, prefix=prefix, timestamp=0)
elif cmd == "poetry":
(runtime, path, poetry_export_extra_args, prefix, tmp_dir) = action[
1:
]
(
runtime,
path,
poetry_export_extra_args,
prefix,
tmp_dir,
pip_install_extra_args,
) = action[1:]
log.info("poetry_export_extra_args: %s", poetry_export_extra_args)
with install_poetry_dependencies(
query, path, poetry_export_extra_args, tmp_dir
query,
path,
poetry_export_extra_args,
tmp_dir,
pip_install_extra_args,
) as rd:
if rd:
if pf:
Expand Down Expand Up @@ -1048,7 +1089,7 @@ def _zip_write_with_filter(


@contextmanager
def install_pip_requirements(query, requirements_file, tmp_dir):
def install_pip_requirements(query, requirements_file, tmp_dir, pip_install_extra_args):
# TODO:
# 1. Emit files instead of temp_dir

Expand Down Expand Up @@ -1125,7 +1166,7 @@ def install_pip_requirements(query, requirements_file, tmp_dir):
"--prefix=",
"--target=.",
"--requirement={}".format(requirements_filename),
]
] + pip_install_extra_args
if docker:
with_ssh_agent = docker.with_ssh_agent
pip_cache_dir = docker.docker_pip_cache
Expand Down Expand Up @@ -1175,7 +1216,9 @@ def install_pip_requirements(query, requirements_file, tmp_dir):


@contextmanager
def install_poetry_dependencies(query, path, poetry_export_extra_args, tmp_dir):
def install_poetry_dependencies(
query, path, poetry_export_extra_args, tmp_dir, pip_install_extra_args
):
# TODO:
# 1. Emit files instead of temp_dir

Expand Down Expand Up @@ -1301,7 +1344,8 @@ def copy_file_to_target(file, temp_dir):
"--prefix=",
"--target=.",
"--requirement=requirements.txt",
],
]
+ pip_install_extra_args,
]
if docker:
with_ssh_agent = docker.with_ssh_agent
Expand Down
Loading