ci(ingest): make ingestion caching rules correct

datahub-project · Aug 22, 2023 · b403519 · b403519
1 parent 8cf299a
commit b403519
Showing 1 changed file with 9 additions and 7 deletions.
diff --git a/metadata-ingestion/build.gradle b/metadata-ingestion/build.gradle
@@ -21,11 +21,13 @@ task checkPythonVersion(type: Exec) {
 }
 
 task environmentSetup(type: Exec, dependsOn: checkPythonVersion) {
+  def sentinel_file = "${venv_name}/.venv_environment_sentinel"
   inputs.file file('setup.py')
-  outputs.dir("${venv_name}")
+  outputs.file(sentinel_file)
   commandLine 'bash', '-c',
     "${python_executable} -m venv ${venv_name} && " +
-    "${venv_name}/bin/python -m pip install --upgrade pip wheel 'setuptools>=63.0.0'"
+    "${venv_name}/bin/python -m pip install --upgrade pip wheel 'setuptools>=63.0.0' && " +
+    "touch ${sentinel_file}"
 }
 
 task runPreFlightScript(type: Exec, dependsOn: environmentSetup) {
@@ -39,17 +41,19 @@ task runPreFlightScript(type: Exec, dependsOn: environmentSetup) {
 task installPackageOnly(type: Exec, dependsOn: runPreFlightScript) {
   def sentinel_file = "${venv_name}/.build_install_package_only_sentinel"
   inputs.file file('setup.py')
-  outputs.dir("${venv_name}")
   outputs.file(sentinel_file)
   commandLine 'bash', '-x', '-c',
     "${venv_name}/bin/pip install -e . &&" +
     "touch ${sentinel_file}"
 }
 
 task installPackage(type: Exec, dependsOn: installPackageOnly) {
+  def sentinel_file = "${venv_name}/.build_install_package_sentinel"
   inputs.file file('setup.py')
-  outputs.dir("${venv_name}")
-  commandLine 'bash', '-x', '-c', "${venv_name}/bin/pip install -e . ${extra_pip_requirements}"
+  outputs.file(sentinel_file)
+  commandLine 'bash', '-x', '-c',
+    "${venv_name}/bin/pip install -e . ${extra_pip_requirements} && " +
+    "touch ${sentinel_file}"
 }
 
 task codegen(type: Exec, dependsOn: [environmentSetup, installPackage, ':metadata-events:mxe-schemas:build']) {
@@ -63,7 +67,6 @@ task install(dependsOn: [installPackage, codegen])
 task installDev(type: Exec, dependsOn: [install]) {
   def sentinel_file = "${venv_name}/.build_install_dev_sentinel"
   inputs.file file('setup.py')
-  outputs.dir("${venv_name}")
   outputs.file(sentinel_file)
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +
@@ -75,7 +78,6 @@ task installDev(type: Exec, dependsOn: [install]) {
 task installAll(type: Exec, dependsOn: [install]) {
   def sentinel_file = "${venv_name}/.build_install_all_sentinel"
   inputs.file file('setup.py')
-  outputs.dir("${venv_name}")
   outputs.file(sentinel_file)
   commandLine 'bash', '-c',
     "source ${venv_name}/bin/activate && set -x && " +