Skip to content

Commit

Permalink
Merge pull request #20 from faldor20/main
Browse files Browse the repository at this point in the history 
Add support for all things null and missing
  • Loading branch information
@lukewilliamboswell
lukewilliamboswell authored Apr 15, 2024
2 parents 3ae34f7 + 843fbc5 commit 35686ac
Show file tree
Hide file tree
Showing 5 changed files with 551 additions and 79 deletions.
236 changes: 157 additions & 79 deletions package/Core.roc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@ interface Core
Json,
json,
jsonWithOptions,
encodeAsNullOption,
]

imports [
Encode.{
Encoder,
Expand All @@ -46,7 +48,7 @@ interface Core

## An opaque type with the `EncoderFormatting` and
## `DecoderFormatting` abilities.
Json := { fieldNameMapping : FieldNameMapping, skipMissingProperties : Bool }
Json := { fieldNameMapping : FieldNameMapping, skipMissingProperties : Bool, nullDecodeAsEmpty : Bool, emptyEncodeAsNull : EncodeAsNull }
implements [
EncoderFormatting {
u8: encodeU8,
Expand Down Expand Up @@ -92,15 +94,41 @@ Json := { fieldNameMapping : FieldNameMapping, skipMissingProperties : Bool }
]

## Returns a JSON `Encoder` and `Decoder`
json = @Json { fieldNameMapping: Default, skipMissingProperties: Bool.true }
json = @Json { fieldNameMapping: Default, skipMissingProperties: Bool.true, nullDecodeAsEmpty: Bool.true, emptyEncodeAsNull: defaultEncodeAsNull }

## Returns a JSON `Encoder` and `Decoder` with configuration options
##
## `skipMissingProperties` - if `True` the decoder will skip additional properties
## **skipMissingProperties** - if `True` the decoder will skip additional properties
## in the json that are not present in the model. (Default: `True`)
jsonWithOptions = \{ fieldNameMapping ? Default, skipMissingProperties ? Bool.true } ->
@Json { fieldNameMapping, skipMissingProperties }
##
## **nullDecodeAsEmpty** - if `True` the decoder will convert `null` to an empty byte array.
## This makes `{"email":null,"name":"bob"}` decode the same as `{"name":"bob"}`. (Default: `True`)
##
## **emptyEncodeAsNull** - if `True` encoders that return `[]` will result in a `null` in the
## json. If `False` when an encoder returns `[]` the record field, or list/tuple element, will be ommitted.
## eg: `{email:@Option None, name:"bob"}` encodes to `{"email":null, "name":"bob"}` instead of `{"name":"bob"}` (Default: `True`)

jsonWithOptions : { fieldNameMapping ? FieldNameMapping, skipMissingProperties ? Bool, nullDecodeAsEmpty ? Bool, emptyEncodeAsNull ? EncodeAsNull } -> Json
jsonWithOptions = \{ fieldNameMapping ? Default, skipMissingProperties ? Bool.true, nullDecodeAsEmpty ? Bool.true, emptyEncodeAsNull ? defaultEncodeAsNull } ->
@Json { fieldNameMapping, skipMissingProperties, nullDecodeAsEmpty, emptyEncodeAsNull }

EncodeAsNull : {
list : Bool,
tuple : Bool,
record : Bool,
}

encodeAsNullOption : { list ? Bool, tuple ? Bool, record ? Bool } -> EncodeAsNull
encodeAsNullOption = \{ list ? Bool.false, tuple ? Bool.true, record ? Bool.true } -> {
list,
tuple,
record,
}
defaultEncodeAsNull = {
list: Bool.false,
tuple: Bool.true,
record: Bool.true,
}
## Mapping between Roc record fields and JSON object names
FieldNameMapping : [
Default, # no transformation
Expand Down Expand Up @@ -283,16 +311,28 @@ expect
actual == expected

encodeList = \lst, encodeElem ->
Encode.custom \bytes, @Json { fieldNameMapping, skipMissingProperties } ->
Encode.custom \bytes, @Json { fieldNameMapping, skipMissingProperties, nullDecodeAsEmpty, emptyEncodeAsNull } ->
writeList = \{ buffer, elemsLeft }, elem ->
bufferWithElem = appendWith buffer (encodeElem elem) (@Json { fieldNameMapping, skipMissingProperties })
bufferWithSuffix =
if elemsLeft > 1 then
List.append bufferWithElem (Num.toU8 ',')
else
bufferWithElem
beforeBufferLen = buffer |> List.len

{ buffer: bufferWithSuffix, elemsLeft: elemsLeft - 1 }
bufferWithElem =
elemBytes =
appendWith [] (encodeElem elem) (@Json { fieldNameMapping, skipMissingProperties, nullDecodeAsEmpty, emptyEncodeAsNull })
|> emptyToNull emptyEncodeAsNull.list
buffer |> List.concat elemBytes

# If our encoder returned [] we just skip the elem
emptyEncode = bufferWithElem |> List.len == beforeBufferLen
if emptyEncode then
{ buffer: bufferWithElem, elemsLeft: elemsLeft - 1 }
else
bufferWithSuffix =
if elemsLeft > 1 then
List.append bufferWithElem (Num.toU8 ',')
else
bufferWithElem

{ buffer: bufferWithSuffix, elemsLeft: elemsLeft - 1 }

head = List.append bytes (Num.toU8 '[')
{ buffer: withList } = List.walk lst { buffer: head, elemsLeft: List.len lst } writeList
Expand All @@ -309,25 +349,35 @@ expect
actual == expected

encodeRecord = \fields ->
Encode.custom \bytes, @Json { fieldNameMapping, skipMissingProperties } ->
Encode.custom \bytes, @Json { fieldNameMapping, skipMissingProperties, nullDecodeAsEmpty, emptyEncodeAsNull } ->
writeRecord = \{ buffer, fieldsLeft }, { key, value } ->

fieldName = toObjectNameUsingMap key fieldNameMapping
fieldValue =
[]
|> appendWith value (@Json { fieldNameMapping, skipMissingProperties, nullDecodeAsEmpty, emptyEncodeAsNull })
|> emptyToNull emptyEncodeAsNull.record

bufferWithKeyValue =
List.append buffer (Num.toU8 '"')
|> List.concat (Str.toUtf8 fieldName)
|> List.append (Num.toU8 '"')
|> List.append (Num.toU8 ':') # Note we need to encode using the json config here
|> appendWith value (@Json { fieldNameMapping, skipMissingProperties })
# If our encoder returned [] we just skip the field

bufferWithSuffix =
if fieldsLeft > 1 then
List.append bufferWithKeyValue (Num.toU8 ',')
else
bufferWithKeyValue

{ buffer: bufferWithSuffix, fieldsLeft: fieldsLeft - 1 }
emptyEncode = fieldValue == []
if emptyEncode then
{ buffer, fieldsLeft: fieldsLeft - 1 }
else
fieldName = toObjectNameUsingMap key fieldNameMapping
bufferWithKeyValue =
List.append buffer (Num.toU8 '"')
|> List.concat (Str.toUtf8 fieldName)
|> List.append (Num.toU8 '"')
|> List.append (Num.toU8 ':') # Note we need to encode using the json config here
|> List.concat fieldValue

bufferWithSuffix =
if fieldsLeft > 1 then
List.append bufferWithKeyValue (Num.toU8 ',')
else
bufferWithKeyValue

{ buffer: bufferWithSuffix, fieldsLeft: fieldsLeft - 1 }

bytesHead = List.append bytes (Num.toU8 '{')
{ buffer: bytesWithRecord } = List.walk fields { buffer: bytesHead, fieldsLeft: List.len fields } writeRecord
Expand Down Expand Up @@ -377,18 +427,27 @@ toYellingCase = \str ->
|> crashOnBadUtf8Error

encodeTuple = \elems ->
Encode.custom \bytes, @Json { fieldNameMapping, skipMissingProperties } ->
Encode.custom \bytes, @Json { fieldNameMapping, skipMissingProperties, nullDecodeAsEmpty, emptyEncodeAsNull } ->
writeTuple = \{ buffer, elemsLeft }, elemEncoder ->
bufferWithElem =
appendWith buffer elemEncoder (@Json { fieldNameMapping, skipMissingProperties })
beforeBufferLen = buffer |> List.len

bufferWithSuffix =
if elemsLeft > 1 then
List.append bufferWithElem (Num.toU8 ',')
else
bufferWithElem
bufferWithElem =
elemBytes =
appendWith [] (elemEncoder) (@Json { fieldNameMapping, skipMissingProperties, nullDecodeAsEmpty, emptyEncodeAsNull })
|> emptyToNull emptyEncodeAsNull.tuple
buffer |> List.concat elemBytes
# If our encoder returned [] we just skip the elem
emptyEncode = bufferWithElem |> List.len == beforeBufferLen
if emptyEncode then
{ buffer: bufferWithElem, elemsLeft: elemsLeft - 1 }
else
bufferWithSuffix =
if elemsLeft > 1 then
List.append bufferWithElem (Num.toU8 ',')
else
bufferWithElem

{ buffer: bufferWithSuffix, elemsLeft: elemsLeft - 1 }
{ buffer: bufferWithSuffix, elemsLeft: elemsLeft - 1 }

bytesHead = List.append bytes (Num.toU8 '[')
{ buffer: bytesWithRecord } = List.walk elems { buffer: bytesHead, elemsLeft: List.len elems } writeTuple
Expand All @@ -404,10 +463,10 @@ expect
actual == expected

encodeTag = \name, payload ->
Encode.custom \bytes, @Json { fieldNameMapping, skipMissingProperties } ->
Encode.custom \bytes, @Json jsonFmt ->
# Idea: encode `A v1 v2` as `{"A": [v1, v2]}`
writePayload = \{ buffer, itemsLeft }, encoder ->
bufferWithValue = appendWith buffer encoder (@Json { fieldNameMapping, skipMissingProperties })
bufferWithValue = appendWith buffer encoder (@Json jsonFmt)
bufferWithSuffix =
if itemsLeft > 1 then
List.append bufferWithValue (Num.toU8 ',')
Expand Down Expand Up @@ -667,7 +726,7 @@ expect
expected = Ok Bool.false
actual.result == expected

decodeTuple = \initialState, stepElem, finalizer -> Decode.custom \initialBytes, @Json {} ->
decodeTuple = \initialState, stepElem, finalizer -> Decode.custom \initialBytes, jsonFmt ->
# NB: the stepper function must be passed explicitly until #2894 is resolved.
decodeElems = \stepper, state, index, bytes ->
{ val: newState, rest: beforeCommaOrBreak } <- tryDecode
Expand All @@ -678,7 +737,8 @@ decodeTuple = \initialState, stepElem, finalizer -> Decode.custom \initialBytes,
{ result: Ok state, rest: beforeCommaOrBreak }

Next decoder ->
Decode.decodeWith (eatWhitespace bytes) decoder json
decodePotentialNull (eatWhitespace bytes) decoder jsonFmt

)

{ result: commaResult, rest: nextBytes } = comma beforeCommaOrBreak
Expand All @@ -691,7 +751,10 @@ decodeTuple = \initialState, stepElem, finalizer -> Decode.custom \initialBytes,

{ val: endStateResult, rest: beforeClosingBracketBytes } <- decodeElems stepElem initialState 0 (eatWhitespace afterBracketBytes) |> tryDecode

{ rest: afterTupleBytes } <- (eatWhitespace beforeClosingBracketBytes) |> closingBracket |> tryDecode
{ rest: afterTupleBytes } <-
(eatWhitespace beforeClosingBracketBytes)
|> closingBracket
|> tryDecode

when finalizer endStateResult is
Ok val -> { result: Ok val, rest: afterTupleBytes }
Expand Down Expand Up @@ -1201,9 +1264,9 @@ expect

# JSON ARRAYS ------------------------------------------------------------------

decodeList = \elemDecoder -> Decode.custom \bytes, @Json { fieldNameMapping, skipMissingProperties } ->
decodeList = \elemDecoder -> Decode.custom \bytes, jsonFmt ->

decodeElems = arrayElemDecoder elemDecoder (@Json { fieldNameMapping, skipMissingProperties })
decodeElems = arrayElemDecoder elemDecoder jsonFmt

result =
when List.walkUntil bytes (BeforeOpeningBracket 0) arrayOpeningHelp is
Expand All @@ -1214,7 +1277,7 @@ decodeList = \elemDecoder -> Decode.custom \bytes, @Json { fieldNameMapping, ski
Ok elemBytes -> decodeElems elemBytes []
Err ExpectedOpeningBracket -> { result: Err TooShort, rest: bytes }

arrayElemDecoder = \elemDecoder, @Json { fieldNameMapping, skipMissingProperties } ->
arrayElemDecoder = \elemDecoder, jsonFmt ->

decodeElems = \bytes, accum ->

Expand All @@ -1238,7 +1301,7 @@ arrayElemDecoder = \elemDecoder, @Json { fieldNameMapping, skipMissingProperties
elemBytes = List.dropFirst bytes n

# Decode current element
{ result, rest } = Decode.decodeWith elemBytes elemDecoder (@Json { fieldNameMapping, skipMissingProperties })
{ result, rest } = decodePotentialNull elemBytes elemDecoder jsonFmt

when result is
Ok elem ->
Expand Down Expand Up @@ -1360,7 +1423,7 @@ expect

# JSON OBJECTS -----------------------------------------------------------------

decodeRecord = \initialState, stepField, finalizer -> Decode.custom \bytes, @Json { fieldNameMapping, skipMissingProperties } ->
decodeRecord = \initialState, stepField, finalizer -> Decode.custom \bytes, @Json { fieldNameMapping, skipMissingProperties, nullDecodeAsEmpty, emptyEncodeAsNull } ->

# Recursively build up record from object field:value pairs
decodeFields = \recordState, bytesBeforeField ->
Expand Down Expand Up @@ -1408,8 +1471,7 @@ decodeRecord = \initialState, stepField, finalizer -> Decode.custom \bytes, @Jso

(Keep valueDecoder, _) ->
# Decode the value using the decoder from the recordState
# Note we need to pass json config options recursively here
Decode.decodeWith valueBytes valueDecoder (@Json { fieldNameMapping, skipMissingProperties })
decodePotentialNull valueBytes valueDecoder (@Json { fieldNameMapping, skipMissingProperties, nullDecodeAsEmpty, emptyEncodeAsNull })
)
|> tryDecode

Expand All @@ -1425,36 +1487,29 @@ decodeRecord = \initialState, stepField, finalizer -> Decode.custom \bytes, @Jso
rest = List.dropFirst bytesAfterValue n

# Build final record from decoded fields and values
when finalizer updatedRecord is
when finalizer updatedRecord json is
##This step is where i can implement my special decoding of options
Ok val -> { result: Ok val, rest }
Err e -> { result: Err e, rest }
Err e ->
{ result: Err e, rest }

_ ->
# Invalid object
{ result: Err TooShort, rest: bytesAfterValue }

countBytesBeforeFirstField =
when List.walkUntil bytes (BeforeOpeningBrace 0) objectHelp is
ObjectFieldNameStart n -> FirstFieldAt n
AfterClosingBrace n -> NoFieldsInObject n
_ -> InvalidObject

when countBytesBeforeFirstField is
InvalidObject ->
# Invalid object, expected opening brace '{' followed by a field
{ result: Err TooShort, rest: bytes }

NoFieldsInObject n ->
# Empty object, try decode using initialState
when finalizer initialState is
Ok val -> { result: Ok val, rest: List.dropFirst bytes n }
Err e -> { result: Err e, rest: bytes }
ObjectFieldNameStart n -> n
_ -> 0

FirstFieldAt n ->
bytesBeforeFirstField = List.dropFirst bytes n
if countBytesBeforeFirstField == 0 then
# Invalid object, expected opening brace '{' followed by a field
{ result: Err TooShort, rest: bytes }
else
bytesBeforeFirstField = List.dropFirst bytes countBytesBeforeFirstField

# Begin decoding field:value pairs
decodeFields initialState bytesBeforeFirstField
# Begin decoding field:value pairs
decodeFields initialState bytesBeforeFirstField

skipFieldHelp : SkipValueState, U8 -> [Break SkipValueState, Continue SkipValueState]
skipFieldHelp = \state, byte ->
Expand Down Expand Up @@ -1508,15 +1563,6 @@ SkipValueState : [
InvalidObject,
]

# Test decode of empty record
expect
input = Str.toUtf8 "{}"

actual : DecodeResult {}
actual = Decode.fromBytesPartial input json

actual.result == Ok {}

# Test decode of partial record
expect
input = Str.toUtf8 "{\"extraField\":2, \"ownerName\": \"Farmer Joe\"}"
Expand Down Expand Up @@ -1699,7 +1745,6 @@ objectHelp = \state, byte ->
(BeforeOpeningBrace n, b) if isWhitespace b -> Continue (BeforeOpeningBrace (n + 1))
(BeforeOpeningBrace n, b) if b == '{' -> Continue (AfterOpeningBrace (n + 1))
(AfterOpeningBrace n, b) if isWhitespace b -> Continue (AfterOpeningBrace (n + 1))
(AfterOpeningBrace n, b) if b == '}' -> Continue (AfterClosingBrace (n + 1))
(AfterOpeningBrace n, b) if b == '"' -> Break (ObjectFieldNameStart n)
(BeforeColon n, b) if isWhitespace b -> Continue (BeforeColon (n + 1))
(BeforeColon n, b) if b == ':' -> Continue (AfterColon (n + 1))
Expand Down Expand Up @@ -2010,3 +2055,36 @@ crashOnBadUtf8Error = \res ->
when res is
Ok str -> str
Err _ -> crash "invalid UTF-8 code units"

nullChars = "null" |> Str.toUtf8

## Returns `Null` if the input starts with "null"
## If makeNullEmpty is true Null{bytes} will be empty
nullToEmpty : List U8, Bool -> [Null _, NotNull]
nullToEmpty = \bytes, makeNullEmpty ->
when bytes is
['n', 'u', 'l', 'l', .. as rest] ->
if makeNullEmpty then
Null { bytes: [], rest }
else
Null { bytes: nullChars, rest }

_ -> NotNull

emptyToNull : List U8, Bool -> List U8
emptyToNull = \bytes, makeEmptyNull ->
if bytes == [] && makeEmptyNull then
nullChars
else
bytes

## If the field value is "null" we may want to make it the same as the field simply not being there for decoding simplicity
decodePotentialNull = \bytes, decoder, @Json jsonFmt ->
when nullToEmpty bytes jsonFmt.nullDecodeAsEmpty is
Null { bytes: nullBytes, rest: nullRest } ->
decode = Decode.decodeWith (nullBytes) decoder (@Json jsonFmt)
# We have to replace the rest because if the null was converted to empty the decoder would return an empty rest
{ result: decode.result, rest: nullRest }

NotNull ->
Decode.decodeWith bytes decoder (@Json jsonFmt)
Loading

0 comments on commit 35686ac

Please sign in to comment.