develop

Signed-off-by: Yicheng-Lu-llll <[email protected]>

Dockerfile

@@ -1,28 +1,11 @@
-ARG PYTHON_VERSION
-FROM python:${PYTHON_VERSION}-slim-buster
-
-MAINTAINER Flyte Team <[email protected]>
-LABEL org.opencontainers.image.source=https://github.com/flyteorg/flytekit
-
+FROM python:3.9-slim-buster
+USER root
 WORKDIR /root
 ENV PYTHONPATH /root
-
-ARG VERSION
-ARG DOCKER_IMAGE
-
 RUN apt-get update && apt-get install build-essential -y
-
-# Pod tasks should be exposed in the default image
-RUN pip install --no-cache-dir -U flytekit==$VERSION \
-	flytekitplugins-pod==$VERSION \
-	flytekitplugins-deck-standard==$VERSION \
-	scikit-learn \
-	&& :
-
-RUN useradd -u 1000 flytekit
-RUN chown flytekit: /root
-# Some packages will create config file under /home by default, so we need to make sure it's writable
-RUN chown flytekit: /home
-USER flytekit
-
-ENV FLYTE_INTERNAL_IMAGE "$DOCKER_IMAGE"
+RUN apt-get install git -y
+# The following line is an example of how to install your modified plugins. In this case, it demonstrates how to install the 'deck' plugin.
+RUN pip install git+https://github.com/Yicheng-Lu-llll/flyte.git@add-ray-autoscaler-config#subdirectory=flyteidl
+RUN pip install -U git+https://github.com/Yicheng-Lu-llll/flytekit.git@"add-ray-autoscaler-config#egg=flytekitplugins-ray&subdirectory=plugins/flytekit-ray" # replace with your own repo and branch
+RUN pip install -U git+https://github.com/Yicheng-Lu-llll/flytekit.git@add-ray-autoscaler-config
+ENV FLYTE_INTERNAL_IMAGE "localhost:30000/flytekit:demo5"

test.py

@@ -0,0 +1,50 @@
+import typing
+
+import ray
+from flytekitplugins.ray import HeadNodeConfig, RayJobConfig, WorkerNodeConfig
+
+from flytekit import Resources, task, workflow
+
+
+@ray.remote
+def f(x):
+    return x * x
+
+
+ray_config = RayJobConfig(
+    head_node_config=HeadNodeConfig(ray_start_params={"log-color": "True"}),
+    worker_node_config=[WorkerNodeConfig(group_name="ray-group", replicas=1)],
+    enable_in_tree_autoscaling=True,
+)
+
+
+@task(
+    task_config=ray_config,
+    requests=Resources(mem="8Gi", cpu="8"),
+)
+def ray_task() -> None:
+    from pprint import pprint
+    import time
+
+    # Import placement group APIs.
+    from ray.util.placement_group import (
+        placement_group,
+        placement_group_table,
+        remove_placement_group,
+    )
+    from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
+
+    # Initialize Ray.
+    import ray
+
+    ray.init()
+
+
+    pg = placement_group([{"CPU": 8} for i in range(1)])
+    ray.get(pg.ready(), timeout=10)
+    print(placement_group_table(pg))
+
+
+@workflow
+def ray_workflow() -> None:
+    return ray_task()

test.sh

@@ -0,0 +1,26 @@
+# Step1: Ensure you have pushed your changes to the remote repo
+# In the flytekit folder
+git add . && git commit -s -m "develop" && git push
+
+# Step2: Build the image
+# In the flytekit folder
+export FLYTE_INTERNAL_IMAGE="localhost:30000/flytekit:demo5" # replace with your own image name and tag
+docker build --no-cache -t  "${FLYTE_INTERNAL_IMAGE}" -f ./Dockerfile .
+
+# Step3: Push the image to the Flyte cluster
+docker push ${FLYTE_INTERNAL_IMAGE}
+
+# Step4: Submit a hello world workflow to the Flyte cluster
+pyflyte run --image ${FLYTE_INTERNAL_IMAGE} --remote /home/ubuntu/flytekit/test.py ray_workflow
+
+
+
+
+# kubectl describe pod  -n flytesnacks-development
+
+#  kubectl get pod -n flytesnacks-development
+
+# kubectl delete rayjob -n flytesnacks-development f64aaaa2db8da46c1a64-n0-0
+
+# kubectl get rayjob -n flytesnacks-development
+# kubectl get rayjob fbb01fe48014e43e383d-n0-0 -n flytesnacks-development -o yaml