adding integration_tests

.github/workflows/main.yml

@@ -41,7 +41,7 @@ jobs:
           test-requirements: "true"
 
       - name: Run black
-        run: black --check dagger tests
+        run: black --check dagger tests integration_tests
 
   flake8:
     runs-on: ubuntu-latest
@@ -129,59 +129,59 @@ jobs:
         with:
           package-import-name: "dagger"
 
-  validate-doc-examples:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v3
-      
-      - name: Validate docs
-        run: ./docker/validate_docs.sh
-
-  build-docs:
-    runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request'
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v3
-
-      - uses: actions/[email protected]
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Install dependencies
-        uses: ./.github/actions/install-dependencies
-        with:
-          requirements: "true"
-          docs-requirements: "true"
-
-      - name: Build Docs
-        run: mkdocs build --strict
-
-      - name: Upload coverage results artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: docs-site
-          path: site/
-
-  update-dev-docs:
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v3
-
-      - uses: actions/[email protected]
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
-
-      - name: Install dependencies
-        uses: ./.github/actions/install-dependencies
-        with:
-          requirements: "true"
-          docs-requirements: "true"
-
-      - name: Push documentation changes
-        uses: ./.github/actions/publish-docs-with-mike
-        with:
-          version_name: dev
+#  validate-doc-examples:
+#    runs-on: ubuntu-latest
+#    steps:
+#      - name: Check out code
+#        uses: actions/checkout@v3
+#
+#      - name: Validate docs
+#        run: ./docker/validate_docs.sh
+#
+#  build-docs:
+#    runs-on: ubuntu-latest
+#    if: github.event_name == 'pull_request'
+#    steps:
+#      - name: Check out code
+#        uses: actions/checkout@v3
+#
+#      - uses: actions/[email protected]
+#        with:
+#          python-version: ${{ env.PYTHON_VERSION }}
+#
+#      - name: Install dependencies
+#        uses: ./.github/actions/install-dependencies
+#        with:
+#          requirements: "true"
+#          docs-requirements: "true"
+#
+#      - name: Build Docs
+#        run: mkdocs build --strict
+#
+#      - name: Upload coverage results artifact
+#        uses: actions/upload-artifact@v3
+#        with:
+#          name: docs-site
+#          path: site/
+#
+#  update-dev-docs:
+#    runs-on: ubuntu-latest
+#    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+#    steps:
+#      - name: Check out code
+#        uses: actions/checkout@v3
+#
+#      - uses: actions/[email protected]
+#        with:
+#          python-version: ${{ env.PYTHON_VERSION }}
+#
+#      - name: Install dependencies
+#        uses: ./.github/actions/install-dependencies
+#        with:
+#          requirements: "true"
+#          docs-requirements: "true"
+#
+#      - name: Push documentation changes
+#        uses: ./.github/actions/publish-docs-with-mike
+#        with:
+#          version_name: dev

Dockerfile

@@ -0,0 +1,30 @@
+FROM python:3.10-buster
+
+ARG _USER="dagger"
+ARG _UID="1001"
+ARG _GID="100"
+ARG _SHELL="/bin/bash"
+
+
+RUN useradd -m -s "${_SHELL}" -N -u "${_UID}" "${_USER}"
+
+ENV USER ${_USER}
+ENV UID ${_UID}
+ENV GID ${_GID}
+ENV HOME /home/${_USER}
+ENV PATH "${HOME}/.local/bin/:${PATH}"
+ENV PIP_NO_CACHE_DIR "true"
+
+
+RUN mkdir /app && chown ${UID}:${GID} /app
+
+USER ${_USER}
+
+COPY --chown=${UID}:${GID} ./requirements*.txt /app/
+WORKDIR /app
+
+RUN pip install -r requirements.txt -r requirements-test.txt
+
+CMD bash
+
+EXPOSE 6066

dagger/templates/template.py

@@ -796,6 +796,7 @@ def build(self) -> TaskTemplate:
 
 
 class IntervalTaskTemplateBuilder(TriggerTaskTemplateBuilder):
+    _time_to_execute_key: Optional[str] = None
     _time_to_force_complete_key: str
     _interval_execute_period_key: str
 

docker-compose.yaml

@@ -0,0 +1,71 @@
+version: '3.5'
+services:
+  dagger_test_app:
+    tty: true
+    build: .
+    container_name: dagger_test_app
+    depends_on:
+      - zookeeper
+      - kafka
+      - aerospike
+    command: [ "./wait-for-it.sh", "kafka:29092", "--", "python3", "-m", "integration_tests.test_app",
+             "worker", "-l", "info" ]
+    environment:
+      - KAFKA_BOOSTRAP_SERVER_NAME=kafka
+      - KAFKA_BOOSTRAP_SERVER_PORT=9092
+      - SCHEMA_REGISTRY_URL=http://schema-registry-server:8081
+      - SCHEMA_REGISTRY_SERVER=schema-registry-server
+      - SCHEMA_REGISTRY_SERVER_PORT=8081
+      - AEROSPIKE_HOST=aerospike
+    ports:
+      - "6066:6066"
+    volumes:
+      - ./:/app
+    networks:
+      - dagger_py
+
+  zookeeper:
+    image: "confluentinc/cp-zookeeper"
+    hostname: zookeeper
+    environment:
+    - ZOOKEEPER_CLIENT_PORT=2181
+    - ZOOKEEPER_TICK_TIME=2000
+    - ZOOKEEPER_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
+    - ZOOKEEPER_TRANSACTION_STATE_LOG_MIN_ISR=1
+    ports:
+    - 52181:2181
+    networks:
+      - dagger_py
+
+  kafka:
+    # pinned due to https://github.com/confluentinc/kafka-images/issues/127
+    image: confluentinc/cp-kafka:7.0.0
+    hostname: kafka
+    container_name: kafka
+    ports:
+    - 9092:9092
+    depends_on:
+    - zookeeper
+    environment:
+      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
+      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
+      - KAFKA_LISTENERS=LISTENER_LOCAL://0.0.0.0:9092,LISTENER_DOCKER://0.0.0.0:29092,PLAINTEXT://0.0.0.0:9093
+      - KAFKA_ADVERTISED_LISTENERS=LISTENER_LOCAL://localhost:9092,LISTENER_DOCKER://kafka:29092,PLAINTEXT://kafka:9093
+      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=LISTENER_LOCAL:PLAINTEXT,LISTENER_DOCKER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+      - KAFKA_BROKER_ID=0
+      - KAFKA_INTER_BROKER_LISTENER_NAME=LISTENER_DOCKER
+      - KAFKA_AUTO_CREATE_TOPICS_ENABLE=true
+      - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=100
+      - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1
+      - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1
+    networks:
+      - dagger_py
+
+  aerospike:
+    image: aerospike/aerospike-server
+    volumes:
+      - $PWD:/opt/aerospike/etc
+    networks:
+      - dagger_py
+networks:
+  dagger_py:

docker/dagger.dockerfile

@@ -1,31 +1,31 @@
-ARG BASE_OS=${BASE_OS:-"BASE_OS NOT SET"}
-ARG DOCKER_PYTHON_VERSION=${DOCKER_PYTHON_VERSION:-"DOCKER_PYTHON_VERSION NOT SET"}
-ARG PYTHON_VERSION=${PYTHON_VERSION:-"PYTHON_VERSION NOT SET"}
-ARG SERVICE_RELEASE_DATE=${SERVICE_RELEASE_DATE:-"SERVICE_RELEASE_DATE NOT SET"}
-
-FROM wayfair/python/${BASE_OS}-${DOCKER_PYTHON_VERSION}-devbox:${PYTHON_VERSION}-${SERVICE_RELEASE_DATE}
-
-ARG PROJECT_NAME=${PROJECT_NAME:-"PROJECT_NAME NOT SET"}
-ARG PROJECT_ROOT_IMPORT_NAME=${PROJECT_ROOT_IMPORT_NAME:-"PROJECT_ROOT_IMPORT_NAME NOT SET"}
-ARG PROJECT_LABEL=${PROJECT_LABEL:-"PROJECT_LABEL NOT SET"}
-ARG PROJECT_MAINTAINER=${PROJECT_MAINTAINER:-"PROJECT_MAINTAINER NOT SET"}
-ARG WF_VERSION
-
-ENV wf_name ${PROJECT_NAME}
-ENV PROJECT_ROOT_IMPORT_NAME ${PROJECT_ROOT_IMPORT_NAME}
-ENV wf_label ${PROJECT_LABEL}
-ENV wf_maintainer ${PROJECT_MAINTAINER}s
-
-# Add image metadata for service image - a requirement for kubernetes deployments
-LABEL \
-  com.wayfair.app=${wf_name} \
-  com.wayfair.description=${wf_label} \
-  com.wayfair.maintainer=${wf_maintainer} \
-  com.wayfair.vendor="Wayfair LLC." \
-  com.wayfair.version=${WF_VERSION}
-
-# 'root' user should be used to install system dependencies, and other things that require elevated permissions
-USER root
+FROM python:3.10-buster
+
+ARG _USER="dagger"
+ARG _UID="1001"
+ARG _GID="100"
+ARG _SHELL="/bin/bash"
+
+
+RUN useradd -m -s "${_SHELL}" -N -u "${_UID}" "${_USER}"
+
+ENV USER ${_USER}
+ENV UID ${_UID}
+ENV GID ${_GID}
+ENV HOME /home/${_USER}
+ENV PATH "${HOME}/.local/bin/:${PATH}"
+ENV PIP_NO_CACHE_DIR "true"
+
+
+RUN mkdir /app && chown ${UID}:${GID} /app
+
+USER ${_USER}
+
+COPY --chown=${UID}:${GID} ./requirements*.txt /app/
+WORKDIR /app
+
+RUN pip install -r requirements.txt -r requirements-test.txt -r requirements-docs.txt
+
+CMD bash
 
 RUN yum -y install snappy lz4