From 060e876cf3876e2f34e6f19705a4b44b08f78fc7 Mon Sep 17 00:00:00 2001 From: Ashley Coleman Date: Fri, 6 Sep 2024 14:33:01 -0600 Subject: [PATCH] address comments --- share/wake/lib/system/io.wake | 10 ++-- share/wake/lib/system/job.wake | 12 ++++ share/wake/lib/system/job_cache_runner.wake | 4 +- .../wake/lib/system/remote_cache_runner.wake | 60 ++++++++----------- 4 files changed, 43 insertions(+), 43 deletions(-) diff --git a/share/wake/lib/system/io.wake b/share/wake/lib/system/io.wake index 24eb5d2db..403a86cef 100644 --- a/share/wake/lib/system/io.wake +++ b/share/wake/lib/system/io.wake @@ -150,9 +150,8 @@ def writeRunner (content: String) = require Pass reality = job.getJobReality # Actually trigger the effects required by the job - def mode = - int smode - | getOrElse 0x200 + require Some mode = int smode + else failWithError "write {path}: Unable to convert mode to Integer ({smode})" require True = mode >= 0 && mode <= 0x1ff else failWithError "write {path}: Invalid mode ({strOctal mode})" @@ -288,9 +287,8 @@ def mkdirRunner: Runner = require Pass reality = job.getJobReality # Actually trigger the effects required by the job - def mode = - int smode - | getOrElse 0x200 + require Some mode = int smode + else failWithError "write {path}: Unable to convert mode to Integer ({smode})" require True = mode >= 0 && mode <= 0x1ff else failWithError "mkdir {path}: Invalid mode ({smode})" diff --git a/share/wake/lib/system/job.wake b/share/wake/lib/system/job.wake index 248206747..263141c1c 100644 --- a/share/wake/lib/system/job.wake +++ b/share/wake/lib/system/job.wake @@ -15,13 +15,25 @@ package wake +# JobKey: The values that the database uses to discern a unique job +# +# If all values of two jobs are identical, then the jobs are considered identical. +# Used to determine reuse eligibility. export tuple JobKey = + # The working directory of the job export dir: String + # A string path to a file which should be passed as the stdin to a job export stdin: String + # The environement that the job runs in export env: String + # The commmand of the job export cmd: String + # A unique hash used to discern two nearly identical jobs with some environmental change export signature: Integer + # The list of files (separated by \0) that the job can see when running export visible: String + # Boolean integer representing if the job should be launched such that it appears to be + # launched directly by a human (ie launched interactively) export isatty: Integer # Create/reserve a job handle, parameters aren't necessarily finalized diff --git a/share/wake/lib/system/job_cache_runner.wake b/share/wake/lib/system/job_cache_runner.wake index ee8473c32..8ed0c1d36 100644 --- a/share/wake/lib/system/job_cache_runner.wake +++ b/share/wake/lib/system/job_cache_runner.wake @@ -178,7 +178,7 @@ export def mkJobCacheRunner (hashFn: RunnerInput => Result String Error) (wakero # Now we need to run the job require Pass (RunnerOutput inputs outputs usage) = baseDoIt job (Pass input) - def Usage status runtime cputime mem ibytes obytes = useage + def Usage status runtime cputime mem ibytes obytes = usage def inputsTree = listToTree scmpCanonical inputs def mkOutputFileJson src = @@ -243,6 +243,6 @@ export def mkJobCacheRunner (hashFn: RunnerInput => Result String Error) (wakero job_cache_add jobCacheAddJson - Pass (RunnerOutput (map getPathName vis) outputs useage) + Pass (RunnerOutput (map getPathName vis) outputs usage) makeRunner "job-cache: {name}" run diff --git a/share/wake/lib/system/remote_cache_runner.wake b/share/wake/lib/system/remote_cache_runner.wake index 6b8f811ac..f53a22437 100644 --- a/share/wake/lib/system/remote_cache_runner.wake +++ b/share/wake/lib/system/remote_cache_runner.wake @@ -56,10 +56,7 @@ export def mkRemoteCacheRunner (rscApi: RemoteCacheApi) (hashFn: RunnerInput => Pass output - def rehydrateJob response label input job = - require (Match details) = response - else unreachable "two-constructor tuple must have one value" - + def rehydrateJob details label input job = def _ = require True = shouldDebugRemoteCache Unit @@ -226,20 +223,8 @@ export def mkRemoteCacheRunner (rscApi: RemoteCacheApi) (hashFn: RunnerInput => require True = rscApi.getRemoteCacheApiCanPull else runJobAndUpload job input hashKey - # ------------------------------------- - # --- Search the cache for the job --- - # ------------------------------------- - - # Search the cache for a match - def requestTask = - rscApi - | rscApiFindMatchingJob (mkSearchRequest input hashKey) - - require Pass response = requestTask - else - require Fail err = requestTask - else unreachable "Result must be either Pass or Fail" - + # If the cache server is down or the response is invalid gracefully fallback + def cacheLookupFailed err = # Always leave a breadcrumb since this should be a rare error. def _ = breadcrumb "{label}: Failed to search for job in the cache" @@ -256,27 +241,32 @@ export def mkRemoteCacheRunner (rscApi: RemoteCacheApi) (hashFn: RunnerInput => # request failed but good to keep in mind. baseDoIt job (Pass input) - # If a match was found use it - require NoMatch = response - else - match (rehydrateJob response label input job) - Pass x -> Pass x - # If the job hydration fails for any reason just run the job as normal. - # There is no point in attempting to push since the server just said its cached - Fail _ -> baseDoIt job (Pass input) + # If a valid response is received from the server then handle it + def cacheLookupSucceeded response = match response + NoMatch -> + def _ = + require True = shouldDebugRemoteCache Unit - def _ = - require True = shouldDebugRemoteCache Unit + def _ = breadcrumb "{label}: Did not find a match" - def _ = breadcrumb "{label}: Did not find a match" - def _ = writeTempFile "remote.cache.lookup.miss" "label: {input.getRunnerInputLabel}" + def _ = + writeTempFile "remote.cache.lookup.miss" "label: {input.getRunnerInputLabel}" - True + True + + # Run the job locally then insert it into the remote cache + runJobAndUpload job input hashKey + Match details -> match (rehydrateJob details label input job) + # If the rehydration succeeds return the job directly + Pass x -> Pass x + # Otherwise if hydration fails for any reason just run the job as normal. + # There is no point in attempting to push since the server just said its cached + Fail _ -> baseDoIt job (Pass input) - # ------------------------------------- - # --- Insert the job into the cache --- - # ------------------------------------- - runJobAndUpload job input hashKey + # Search the remote cache for the job + match (rscApi | rscApiFindMatchingJob (mkSearchRequest input hashKey)) + Pass response -> cacheLookupSucceeded response + Fail err -> cacheLookupFailed err makeRunner "remote-cache: {name}" run