From 6b7cd100549c5f16b615c30c9f73806a971d1afc Mon Sep 17 00:00:00 2001 From: Anna Geller Date: Sun, 17 Sep 2023 21:04:56 +0200 Subject: [PATCH] feat(docs): extend the trigger page to explain variable usage and event consumption (#513) --- content/docs/02.tutorial/04.triggers.md | 122 ++++++++++++++++-------- 1 file changed, 80 insertions(+), 42 deletions(-) diff --git a/content/docs/02.tutorial/04.triggers.md b/content/docs/02.tutorial/04.triggers.md index 92324fbf10..0dcb80b1f5 100644 --- a/content/docs/02.tutorial/04.triggers.md +++ b/content/docs/02.tutorial/04.triggers.md @@ -2,83 +2,121 @@ title: Triggers --- +You can use triggers to **automatically** start a flow based on an event. Kestra supports both **scheduled** and **external events**. Kestra core provides three types of triggers: -You use triggers to start a flow. They can be scheduled or triggered by an event. -Kestra core provides three types of triggers: +* [Schedule trigger](../05.developer-guide/08.triggers/01.schedule.md) allows you to trigger your flow on a regular cadence e.g. using a schedule defined by a CRON expression +* [Flow trigger](../05.developer-guide/08.triggers/02.flow.md) allows you to trigger your flow when another flow finishes its execution (based on a configurable list of states) +* [Webhook trigger](../05.developer-guide/08.triggers/03.webhook.md) allows you to trigger your flow based on an HTTP request emitted by a webhook. -* [Schedule](../05.developer-guide/08.triggers/01.schedule.md) - * Trigger your flow based on a cron expression -* [Flow trigger](../05.developer-guide/08.triggers/02.flow.md) - * Trigger your flow based on another flow end -* [Webhook](../05.developer-guide/08.triggers/03.webhook.md) - * Trigger your flow based on an HTTP request +Many other triggers are available from the plugins, such as triggers based on file detection events, e.g. the [S3 trigger](https://kestra.io/plugins/plugin-aws/triggers/s3/io.kestra.plugin.aws.s3.trigger), or a new message arrival in a message queue, such as the [SQS](https://kestra.io/plugins/plugin-aws/triggers/sqs/io.kestra.plugin.aws.sqs.trigger) or [Kafka trigger](https://kestra.io/plugins/plugin-kafka/triggers/io.kestra.plugin.kafka.trigger). -Many other triggers are available from the plugins, such as triggers based on file creation or a message queue. +--- + +## Trigger variables + +Triggers allow you to access trigger metadata through [Variables](../05.developer-guide/03.variables/01.index.md) e.g. `{{ trigger.date }}` to access the current date of the [Schedule trigger](https://kestra.io/plugins/core/triggers/io.kestra.core.models.triggers.types.schedule), `{{ trigger.uri }}` to access the file or message from any file detection or message arrival event, as well as `{{ trigger.rows }}` for all Query triggers e.g. the [PostgreSQL Query](https://kestra.io/plugins/plugin-jdbc-postgres/triggers/io.kestra.plugin.jdbc.postgresql.trigger) trigger. + +::alert{type="warning"} +Note that the above-mentioned **templated variables** are only available when the execution is created **automatically** by the trigger. You'll get an error if you try to run a flow containing such variables **manually**. + +Also, note that **you don't need an extra task to consume** the file or message from the event. Kestra downloads those automatically to the **internal storage** and makes those available in your flow using `{{ trigger.uri }}` variable. Therefore, you don't need any additional task to e.g. consume a message from the SQS queue or to download a file from S3 when using those event triggers. The trigger already consumes and downloads those, making them directly available for further processing. Check the documentation of a specific trigger and [Blueprints](../04.user-interface-guide/blueprints.md) with the **Trigger** tag for more details and examples. +:: + +--- ## Defining triggers -The Trigger definition is close to the task definition. We define them in the `triggers` section of the flow. And as a Task, it contains an `id`, a `type`, and some properties related to his type. A Flow can have multiple Triggers. +Use the `triggers` keyword in the flow and deifne a list of triggers — you can have several triggers attached to a flow. + +The `trigger` definition looks similar to the task definition — it contains an `id`, a `type`, and additional properties related to the specific trigger type. -The below triggers will start the flow every day at 10 am or when the `trigger-flow` flow ends. +The workflow below will be automatically triggered every day at 10 AM, as well as anytime when the `first_flow` finishes its execution. Both triggers are independent of each other. ```yaml +id: myflow +namespace: dev + + +tasks: + - id: hello + type: io.kestra.core.tasks.log.Log + message: Have a great day! + + triggers: - - id: schedule + - id: schedule_trigger type: io.kestra.core.models.triggers.types.Schedule cron: 0 10 * * * - - id: listenFlow + + + - id: flow_trigger type: io.kestra.core.models.triggers.types.Flow conditions: - type: io.kestra.core.models.conditions.types.ExecutionFlowCondition - namespace: io.kestra.tutorial - flowId: trigger-flow + namespace: dev + flowId: first_flow ``` +--- + ## Add a trigger to your flow -Let's schedule our flow. This trigger will start our flow every Monday at 10 am. +Let's look at another trigger example. This trigger will start our flow every Monday at 10 AM. ```yaml triggers: - - id: schedule + - id: every_monday_at_10_am type: io.kestra.core.models.triggers.types.Schedule cron: 0 10 * * 1 ``` -::collapse{title="Click here to see the full flow"} +::collapse{title="Click here to see the full workflow example with this Schedule trigger"} ```yaml -id: kestra-tutorial -namespace: io.kestra.tutorial +id: tutorial +namespace: dev + labels: - env: PRD + env: prod + description: | # Kestra Tutorial - As you notice, we can use markdown here. + We can use `markdown` here. + tasks: - - id: download - type: io.kestra.plugin.fs.http.Download - uri: "https://gist.githubusercontent.com/tchiotludo/2b7f28f4f507074e60150aedb028e074/raw/6b6348c4f912e79e3ffccaf944fd019bf51cba30/conso-elec-gaz-annuelle-par-naf-agregee-region.csv" - - id: analyze-data - type: io.kestra.core.tasks.scripts.Python - runner: DOCKER - dockerOptions: - image: python + - id: api + type: io.kestra.plugin.fs.http.Request + uri: https://dummyjson.com/products + + - id: python + type: io.kestra.plugin.scripts.python.Script + docker: + image: python:slim + beforeCommands: + - pip install polars + warningOnStdErr: false + script: | + import polars as pl + data = {{outputs.api.body | jq('.products') | first}} + df = pl.from_dicts(data) + df.glimpse() + df.select(["brand", "price"]).write_csv("{{outputDir}}/products.csv") + + - id: sql_query + type: io.kestra.plugin.jdbc.duckdb.Query inputFiles: - data.csv: "{{outputs.download.uri}}" - main.py: | - import pandas as pd - from kestra import Kestra - data = pd.read_csv("data.csv", sep=";") - data.info() - sumOfConsumption = data['conso'].sum() - Kestra.outputs({'sumOfConsumption': int(sumOfConsumption)}) - requirements: - - pandas + in.csv: "{{ outputs.python.outputFiles['products.csv'] }}" + sql: | + SELECT brand, round(avg(price), 2) as avg_price + FROM read_csv_auto('{{workingDir}}/in.csv', header=True) + GROUP BY brand + ORDER BY avg_price DESC; + store: true + triggers: - - id: schedule + - id: every_monday_at_10_am type: io.kestra.core.models.triggers.types.Schedule - cron: 0 10 * * * + cron: 0 10 * * 1 ``` ::