From db817e8a0fcb9279a55e04fa19e05eeeaae15aa1 Mon Sep 17 00:00:00 2001 From: Future-Outlier Date: Thu, 7 Dec 2023 06:15:17 +0800 Subject: [PATCH] Sensor Agent Doc Integration Example (#1180) * sensor agent doc Signed-off-by: Future Outlier * sensor example Signed-off-by: Future Outlier * new line Signed-off-by: Future Outlier * lint Signed-off-by: Future Outlier * Update examples/sensor/sensor/sensor.py Co-authored-by: Kevin Su Signed-off-by: Future-Outlier * Update examples/sensor/sensor/sensor.py Co-authored-by: Kevin Su Signed-off-by: Future-Outlier * Update examples/sensor/sensor/sensor.py Co-authored-by: Kevin Su Signed-off-by: Future-Outlier * Update examples/sensor/sensor/sensor.py Co-authored-by: Kevin Su Signed-off-by: Future-Outlier * Update examples/sensor/sensor/sensor.py Co-authored-by: Kevin Su Signed-off-by: Future-Outlier * Update examples/sensor/sensor/sensor.py Co-authored-by: Kevin Su Signed-off-by: Future-Outlier * Update examples/sensor/sensor/sensor.py Co-authored-by: Kevin Su Signed-off-by: Future-Outlier * Update examples/sensor/sensor/sensor.py Co-authored-by: Kevin Su Signed-off-by: Future-Outlier * Update examples/sensor/sensor/sensor.py Co-authored-by: Kevin Su Signed-off-by: Future-Outlier * Update examples/sensor/README.md Co-authored-by: Kevin Su Signed-off-by: Future-Outlier * Update examples/sensor/sensor/sensor.py Co-authored-by: Kevin Su Signed-off-by: Future-Outlier * Update examples/sensor/sensor/sensor.py Co-authored-by: Kevin Su Signed-off-by: Future-Outlier * add file_sensor example command line Signed-off-by: Future Outlier * use neverett's advice Signed-off-by: Future Outlier * fix doc error Signed-off-by: Future Outlier * Trigger CI Signed-off-by: Future Outlier * remove $ Signed-off-by: Future Outlier * fixed! Signed-off-by: Future Outlier --------- Signed-off-by: Future Outlier Signed-off-by: Future-Outlier Co-authored-by: Future Outlier Co-authored-by: Kevin Su Co-authored-by: Kevin Su --- docs/index.md | 1 + docs/integrations.md | 2 + examples/sensor/README.md | 20 +++++++++ examples/sensor/sensor/__init__.py | 0 examples/sensor/sensor/file_sensor_example.py | 44 +++++++++++++++++++ 5 files changed, 67 insertions(+) create mode 100644 examples/sensor/README.md create mode 100644 examples/sensor/sensor/__init__.py create mode 100644 examples/sensor/sensor/file_sensor_example.py diff --git a/docs/index.md b/docs/index.md index 0ebdc325b..d94148a3f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -456,6 +456,7 @@ auto_examples/papermill_plugin/index auto_examples/pandera_plugin/index auto_examples/kfpytorch_plugin/index auto_examples/ray_plugin/index +auto_examples/sensor/index auto_examples/snowflake_plugin/index auto_examples/k8s_spark_plugin/index auto_examples/sql_plugin/index diff --git a/docs/integrations.md b/docs/integrations.md index 83bb5e878..de04dded8 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -106,6 +106,8 @@ the Flyte task that use the respective plugin. - Run Hive jobs in your workflows. * - {doc}`MMCloud ` - Execute tasks using MemVerge Memory Machine Cloud +* - {doc}`Sensor ` + - Run Sensor jobs in your workflows. * - {doc}`Snowflake ` - Run Snowflake jobs in your workflows. * - {doc}`Databricks ` diff --git a/examples/sensor/README.md b/examples/sensor/README.md new file mode 100644 index 000000000..9bbcb5f81 --- /dev/null +++ b/examples/sensor/README.md @@ -0,0 +1,20 @@ +(sensor)= + +# Sensor + +```{eval-rst} +.. tags:: Data, Basic +``` + +## Run the example on the Flyte cluster + +To run the provided example on the Flyte cluster, use the following command: + +``` +pyflyte run --remote \ + https://raw.githubusercontent.com/flyteorg/flytesnacks/master/examples/sensor/sensor/file_sensor_example.py wf +``` + +```{auto-examples-toc} +file_sensor_example +``` diff --git a/examples/sensor/sensor/__init__.py b/examples/sensor/sensor/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/sensor/sensor/file_sensor_example.py b/examples/sensor/sensor/file_sensor_example.py new file mode 100644 index 000000000..aa82b1cbc --- /dev/null +++ b/examples/sensor/sensor/file_sensor_example.py @@ -0,0 +1,44 @@ +# %% [markdown] +# # File Sensor +# +# This example shows how to use the `FileSensor` to detect files appearing in your local or remote filesystem. +# +# First, import the required libraries. + +# %% +from flytekit import task, workflow +from flytekit.sensor.file_sensor import FileSensor + +# %% [markdown] +# Next, create a FileSensor task. + +# %% +sensor = FileSensor(name="test_file_sensor") + +# %% [markdown] +# To use the FileSensor created in the previous step, you must specify the path parameter. In the sandbox, you can use the S3 path. + + +# %% +@task() +def t1(): + print("SUCCEEDED") + + +@workflow() +def wf(): + sensor(path="s3://my-s3-bucket/file.txt") >> t1() + + +if __name__ == "__main__": + wf() + +# %% [markdown] +# You can also use the S3 or GCS file system. +# We have already set the minio credentials in the agent by default. If you test the sandbox example locally, you will need to set the AWS credentials in your environment variables. +# +# ```{prompt} bash +# export FLYTE_AWS_ENDPOINT="http://localhost:30002" +# export FLYTE_AWS_ACCESS_KEY_ID="minio" +# export FLYTE_AWS_SECRET_ACCESS_KEY="miniostorage" +# ```