From 2d957262c38e6ebbd8db495cbd00637c3e342fe2 Mon Sep 17 00:00:00 2001 From: Ashley Coleman Date: Fri, 3 May 2024 09:37:06 -0700 Subject: [PATCH] rsc: Support dbonly small blobs on the client (#1553) * rsc: Support dbonly small blobs on the client * address comments * fix off by one * refine size --- share/wake/lib/system/http.wake | 15 +++-- share/wake/lib/system/remote_cache_api.wake | 65 ++++++++++++++++----- 2 files changed, 62 insertions(+), 18 deletions(-) diff --git a/share/wake/lib/system/http.wake b/share/wake/lib/system/http.wake index d7e5d7c66..30bda5114 100644 --- a/share/wake/lib/system/http.wake +++ b/share/wake/lib/system/http.wake @@ -31,6 +31,8 @@ tuple HttpFormData = # in the fullness of time it should be converted to a Path. This library is careful to not # claim or otherwise convert it to a Path as it should be done by the caller when possible File: String + # The optional Content-Type field of the specific form entry. The field is not set when None + ContentType: Option String # The description of a http request to send to a web server. tuple HttpRequest = @@ -104,15 +106,15 @@ export def setBody (body: String): HttpRequest => HttpRequest = setHttpRequestBody (Some body) # Adds a named file to the requests form data. -export def addFormData (name: String) (value: Path): HttpRequest => HttpRequest = - unsafe_addFormData name value.getPathName +export def addFormData (name: String) (value: Path) (contentType: Option String): HttpRequest => HttpRequest = + unsafe_addFormData name value.getPathName contentType # Adds a named file to the request's form data from an unhashed file. # # This is a restricted function and should be reserved for advanced use only -export def unsafe_addFormData (name: String) (value: String): HttpRequest => HttpRequest = +export def unsafe_addFormData (name: String) (value: String) (contentType: Option String): HttpRequest => HttpRequest = def setOrAppend field = - def entry = HttpFormData name value + def entry = HttpFormData name value contentType match field Some formDatas -> Some (entry, formDatas) @@ -192,7 +194,10 @@ def methodToString = match _ # helper function for building up the curl cmd representing a request def makeCurlCmd ((HttpRequest url method headers body formData): HttpRequest) (extraFlags: List String): Result (List String) Error = def headerToCurlFlag (HttpHeader name value) = "--header", "{name}:{value}", Nil - def formDataToCurlFlag (HttpFormData name file) = "--form", "{name}=@{file}", Nil + + def formDataToCurlFlag (HttpFormData name file contentType) = match contentType + Some ct -> "--form", "{name}=@{file};type={ct}", Nil + None -> "--form", "{name}=@{file}", Nil def bodyToCurlFlag body = require True = body.strlen >= 5000 diff --git a/share/wake/lib/system/remote_cache_api.wake b/share/wake/lib/system/remote_cache_api.wake index a611a8a12..ac6c2ac36 100644 --- a/share/wake/lib/system/remote_cache_api.wake +++ b/share/wake/lib/system/remote_cache_api.wake @@ -237,13 +237,15 @@ export def makeRemoteCacheApi (config: String): Result RemoteCacheApi Error = # (RemoteCacheApi "foo" 1 None) | rscApiPostStringBlob "foo" "my foo contents" = Fail "authorization required" # ``` export def rscApiPostStringBlob (name: String) (value: String) (api: RemoteCacheApi): Result String Error = - # TODO: use "small blob" for this - # require False = value ==* "" - # else Pass "00000000-0000-0000-0000-000000000000" + def contentType = + if value.strlen < 95 then + Some "blob/small" + else + None require Pass temp = writeTempFile name value - uploadBlobRequest api (addFormData name temp) + uploadBlobRequest api (addFormData name temp contentType) # rscApiPostFileBlob: Posts a named file on disk to the remote server defined by *api* # then returns the id associated to the blob. Requires authorization. @@ -255,7 +257,7 @@ export def rscApiPostStringBlob (name: String) (value: String) (api: RemoteCache export def rscApiPostFileBlob (name: String) (file: String) (api: RemoteCacheApi): Result String Error = # We must use unsafe here since we cannot elevate *file* to a Path without either copying it # or triggering the 'job output by multiple files' error. - uploadBlobRequest api (unsafe_addFormData name file) + uploadBlobRequest api (unsafe_addFormData name file None) # rscApiPostJob: Posts a job defined by *req* to the remote cache server. Requires authorization. # @@ -395,16 +397,53 @@ export def rscApiCheckAuthorization (api: RemoteCacheApi): Result Unit Error = # rscApiGetStringBlob (RemoteCacheBlob "asdf" "https://...") = Pass "foo\nbar\nbat" # ``` export def rscApiGetStringBlob ((CacheSearchBlob _ uri): CacheSearchBlob): Result String Error = - # TODO: use "small blob" for this and also parse the schema out - # require False = id ==* "00000000-0000-0000-0000-000000000000" - # else Pass "" + require scheme, path, Nil = extract `([a-zA-Z][a-zA-Z0-9+.-]*)://(.*)` uri + else failWithError "rsc: uri has unexpected format: '{uri}'" + + # maps path = "%46%6F%6F" to content = "foo" + def dbScheme _scheme path = + # If path is empty we have the empty string + require True = path.strlen > 0 + else Pass "" + + # If not empty then it must as least be %00 + require True = path.strlen >= 3 + else failWithError "rsc: Invalid db path: '{path}'" + + require "", byteStrs = tokenize `%` path + else failWithError "rsc: Failed to extract bytes from db path: '{path}'" + + require Some bytes = + byteStrs + | map (intbase 16) + | findNone + else failWithError "rsc: Failed to parse bytes strings into bytes from db path: '{path}'" + + bytes + # TODO: this only supports ASCII, not unicode. This needs to be enforced somewhere. + | map integerToByte + | cat + | Pass - require Pass response = - buildHttpRequest uri - | setMethod HttpMethodGet - | makeRequest + def fileScheme scheme path = + require Pass response = + buildHttpRequest "{scheme}://{path}" + | setMethod HttpMethodGet + | makeRequest + + Pass response.getHttpResponseBody + + def unsupportedScheme scheme path = + failWithError "rsc: scheme '{scheme}' from uri '{uri}' is not supported" + + def schemeFn = match scheme + "db" -> dbScheme + "file" -> fileScheme + "http" -> fileScheme + "https" -> fileScheme + _ -> unsupportedScheme - Pass response.getHttpResponseBody + schemeFn scheme path # rscApiGetFileBlob: Downloads a blob to *path* with *mode* permisssions and return *path* #