diff --git a/Dockerfile b/Dockerfile index 87515506dd..92379f221e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,11 @@ -ARG PYTHON_VERSION -FROM python:${PYTHON_VERSION}-slim-buster - -MAINTAINER Flyte Team -LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit - +FROM python:3.9-slim-buster +USER root WORKDIR /root ENV PYTHONPATH /root - -ARG VERSION -ARG DOCKER_IMAGE - RUN apt-get update && apt-get install build-essential -y - -# Pod tasks should be exposed in the default image -RUN pip install --no-cache-dir -U flytekit==$VERSION \ - flytekitplugins-pod==$VERSION \ - flytekitplugins-deck-standard==$VERSION \ - scikit-learn \ - && : - -RUN useradd -u 1000 flytekit -RUN chown flytekit: /root -# Some packages will create config file under /home by default, so we need to make sure it's writable -RUN chown flytekit: /home -USER flytekit - -ENV FLYTE_INTERNAL_IMAGE "$DOCKER_IMAGE" +RUN apt-get install git -y +# The following line is an example of how to install your modified plugins. In this case, it demonstrates how to install the 'deck' plugin. +RUN pip install git+https://github.com/Yicheng-Lu-llll/flyte.git@add-ray-autoscaler-config#subdirectory=flyteidl +RUN pip install -U git+https://github.com/Yicheng-Lu-llll/flytekit.git@"add-ray-autoscaler-config#egg=flytekitplugins-ray&subdirectory=plugins/flytekit-ray" # replace with your own repo and branch +RUN pip install -U git+https://github.com/Yicheng-Lu-llll/flytekit.git@add-ray-autoscaler-config +ENV FLYTE_INTERNAL_IMAGE "localhost:30000/flytekit:demo5" diff --git a/bin/flytectl b/bin/flytectl new file mode 100755 index 0000000000..f34db3b3dd Binary files /dev/null and b/bin/flytectl differ diff --git a/test.py b/test.py new file mode 100644 index 0000000000..4fd0e1f919 --- /dev/null +++ b/test.py @@ -0,0 +1,50 @@ +import typing + +import ray +from flytekitplugins.ray import HeadNodeConfig, RayJobConfig, WorkerNodeConfig + +from flytekit import Resources, task, workflow + + +@ray.remote +def f(x): + return x * x + + +ray_config = RayJobConfig( + head_node_config=HeadNodeConfig(ray_start_params={"log-color": "True"}), + worker_node_config=[WorkerNodeConfig(group_name="ray-group", replicas=1)], + enable_in_tree_autoscaling=True, +) + + +@task( + task_config=ray_config, + requests=Resources(mem="8Gi", cpu="8"), +) +def ray_task() -> None: + from pprint import pprint + import time + + # Import placement group APIs. + from ray.util.placement_group import ( + placement_group, + placement_group_table, + remove_placement_group, + ) + from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy + + # Initialize Ray. + import ray + + ray.init() + + + pg = placement_group([{"CPU": 8} for i in range(1)]) + ray.get(pg.ready(), timeout=10) + print(placement_group_table(pg)) + + +@workflow +def ray_workflow() -> None: + return ray_task() diff --git a/test.sh b/test.sh new file mode 100755 index 0000000000..edae6c2325 --- /dev/null +++ b/test.sh @@ -0,0 +1,26 @@ +# Step1: Ensure you have pushed your changes to the remote repo +# In the flytekit folder +git add . && git commit -s -m "develop" && git push + +# Step2: Build the image +# In the flytekit folder +export FLYTE_INTERNAL_IMAGE="localhost:30000/flytekit:demo5" # replace with your own image name and tag +docker build --no-cache -t "${FLYTE_INTERNAL_IMAGE}" -f ./Dockerfile . + +# Step3: Push the image to the Flyte cluster +docker push ${FLYTE_INTERNAL_IMAGE} + +# Step4: Submit a hello world workflow to the Flyte cluster +pyflyte run --image ${FLYTE_INTERNAL_IMAGE} --remote /home/ubuntu/flytekit/test.py ray_workflow + + + + +# kubectl describe pod -n flytesnacks-development + +# kubectl get pod -n flytesnacks-development + +# kubectl delete rayjob -n flytesnacks-development f64aaaa2db8da46c1a64-n0-0 + +# kubectl get rayjob -n flytesnacks-development +# kubectl get rayjob fbb01fe48014e43e383d-n0-0 -n flytesnacks-development -o yaml