add version pinning

datahub-project · Nov 17, 2023 · ab30dae · ab30dae
1 parent 97fa4a0
commit ab30dae
Show file tree

Hide file tree

Showing 8 changed files with 22 additions and 7 deletions.
diff --git a/docker/docker-compose-without-neo4j.override.yml b/docker/docker-compose-without-neo4j.override.yml
@@ -48,7 +48,7 @@ services:
   mysql:
     container_name: mysql
     hostname: mysql
-    image: mysql:8.2
+    image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
     command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
     ports:
       - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306

diff --git a/docker/docker-compose.override.yml b/docker/docker-compose.override.yml
@@ -32,7 +32,7 @@ services:
   mysql:
     container_name: mysql
     hostname: mysql
-    image: mysql:8.2
+    image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
     command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
     ports:
     - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306

diff --git a/docker/mysql/docker-compose.mysql.yml b/docker/mysql/docker-compose.mysql.yml
@@ -5,7 +5,7 @@ services:
   mysql:
     container_name: mysql
     hostname: mysql
-    image: mysql:8.2
+    image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
     env_file: env/docker.env
     command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
     ports:

diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml
@@ -218,7 +218,7 @@ services:
       test: mysqladmin ping -h mysql -u $$MYSQL_USER --password=$$MYSQL_PASSWORD
       timeout: 5s
     hostname: mysql
-    image: mysql:8.2
+    image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
     ports:
     - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
     restart: on-failure

diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml
@@ -225,7 +225,7 @@ services:
       test: mysqladmin ping -h mysql -u $$MYSQL_USER --password=$$MYSQL_PASSWORD
       timeout: 5s
     hostname: mysql
-    image: mysql:8.2
+    image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
     ports:
     - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
     restart: on-failure

diff --git a/docker/quickstart/quickstart_version_mapping.yaml b/docker/quickstart/quickstart_version_mapping.yaml
@@ -23,18 +23,27 @@ quickstart_version_map:
   default:
     composefile_git_ref: master
     docker_tag: head
+    mysql_tag: 5.7
   # default:  # Use this to pin default to a specific version.
   #   composefile_git_ref: fd1bd51541a132017a648f4a2f037eec8f70ba26 # v0.10.0 + quickstart compose file fixes
   #   docker_tag: v0.10.0
 
   head:
     composefile_git_ref: master
     docker_tag: head
+    mysql_tag: 5.7
+
+  # v0.13.0 we upgraded MySQL image for EOL
+  v0.13.0:
+    composefile_git_ref: master
+    docker_tag: head
+    mysql_tag: 8.2
 
   # v0.9.6 images contain security vulnerabilities
   v0.9.6:
     composefile_git_ref: v0.9.6.1
     docker_tag: v0.9.6.1
+    mysql_tag: 5.7
 
   # If stable is not defined the latest released version will be used.
   # stable:

diff --git a/metadata-ingestion/src/datahub/cli/docker_cli.py b/metadata-ingestion/src/datahub/cli/docker_cli.py
@@ -158,6 +158,7 @@ def should_use_neo4j_for_graph_service(graph_service_override: Optional[str]) ->
 
 def _set_environment_variables(
     version: Optional[str],
+    mysql_version: Optional[str],
     mysql_port: Optional[pydantic.PositiveInt],
     zk_port: Optional[pydantic.PositiveInt],
     kafka_broker_port: Optional[pydantic.PositiveInt],
@@ -172,6 +173,8 @@ def _set_environment_variables(
             )
             version = f"v{version}"
         os.environ["DATAHUB_VERSION"] = version
+    if mysql_version is not None:
+        os.environ["DATAHUB_MYSQL_VERSION"] = mysql_version
     if mysql_port is not None:
         os.environ["DATAHUB_MAPPED_MYSQL_PORT"] = str(mysql_port)
 
@@ -675,6 +678,7 @@ def quickstart(  # noqa: C901
     # set version
     _set_environment_variables(
         version=quickstart_execution_plan.docker_tag,
+        mysql_version=quickstart_execution_plan.mysql_tag,
         mysql_port=mysql_port,
         zk_port=zk_port,
         kafka_broker_port=kafka_broker_port,

diff --git a/metadata-ingestion/src/datahub/cli/quickstart_versioning.py b/metadata-ingestion/src/datahub/cli/quickstart_versioning.py
@@ -21,6 +21,7 @@
 class QuickstartExecutionPlan(BaseModel):
     composefile_git_ref: str
     docker_tag: str
+    mysql_tag: str
 
 
 def _is_it_a_version(version: str) -> bool:
@@ -81,7 +82,7 @@ def fetch_quickstart_config(cls) -> "QuickstartVersionMappingConfig":
             return QuickstartVersionMappingConfig(
                 quickstart_version_map={
                     "default": QuickstartExecutionPlan(
-                        composefile_git_ref="master", docker_tag="head"
+                        composefile_git_ref="master", docker_tag="head", mysql_tag="5.7"
                     ),
                 }
             )
@@ -114,10 +115,11 @@ def get_quickstart_execution_plan(
             requested_version = "default"
         composefile_git_ref = requested_version
         docker_tag = requested_version
+        mysql_tag = requested_version
         result = self.quickstart_version_map.get(
             requested_version,
             QuickstartExecutionPlan(
-                composefile_git_ref=composefile_git_ref, docker_tag=docker_tag
+                composefile_git_ref=composefile_git_ref, docker_tag=docker_tag, mysql_tag=mysql_tag
             ),
         )
         # new CLI version is downloading the composefile corresponding to the requested version