From aaea97c2d2b71451301f189814f65ba59e6bbe96 Mon Sep 17 00:00:00 2001 From: tan Date: Mon, 11 Oct 2021 10:51:48 +0530 Subject: [PATCH] Support chunked transfer encoding Handle APIs that return chunked transfer encoded data. E.g. Kubernetes `watch` APIs. Fixes #41 --- README.md | 26 ++++-- plugin/build.sh | 2 +- plugin/pom.xml | 2 +- plugin/src/main/resources/julia/api.mustache | 12 ++- src/client.jl | 88 +++++++++++++++++--- test/petstore/test_StoreApi.jl | 15 ++++ 6 files changed, 121 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 111223b..ad6f3ee 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ First, you need to build the Swagger Java libraries. Ensure you have Java and ma plugin/build.sh ``` -A single jar file (julia-swagger-codegen-0.0.3.jar) will be produced in `plugin/target`. +A single jar file (julia-swagger-codegen-0.0.4.jar) will be produced in `plugin/target`. You can now use that for codegen. @@ -32,7 +32,7 @@ Note: problems have been reported while building with JDK 9 on MacOS likely beca Use the supplied script `plugin/generate.sh` and point it to the specification file and a configuration file. E.g.: -``` +```bash ${SWAGGERDIR}/plugin/generate.sh -i ${SPECDIR}/${SPECFILE} -o ${GENDIR} -c config.json ``` _where_ @@ -51,16 +51,26 @@ The configuration file (`config.json`) can have the following options: ### APIs -Each API set is generated into a file named `api_.jl`. It is represented as a `struct` and the APIs under it are generated as methods. An API set can be constructed by providing the -swagger client instance that it can use for communication. +Each API set is generated into a file named `api_.jl`. It is represented as a `struct` and the APIs under it are generated as methods. An API set can be constructed by providing the swagger client instance that it can use for communication. + +The required API parameters are generated as regular function arguments. Optional parameters are generated as keyword arguments. Method documentation is generated with description, parameter information and return value. Two variants of the API are generated. The first variant is suitable for calling synchronously and returns a single instance of the result struct. + +```julia +# example synchronous API that returns an Order instance +getOrderById(api::StoreApi, orderId::Int64) +``` + +The second variant is suitable for asynchronous calls to methods that return chunked transfer encoded responses, where in the API streams the response objects into an output channel. -The required API parameters are generated as regular function arguments. Optional parameters are generated as keyword arguments. Method -documentation is generated with description, parameter information and return value. +```julia +# example asynchronous API that streams matching Pet instances into response_stream +findPetsByStatus(api::PetApi, response_stream::Channel, status::Vector{String}) +``` A client context holds common information to be used across APIs. It also holds a connection to the server and uses that across API calls. The client context needs to be passed as the first parameter of all API calls. It can be created as: -``` +```julia Client(root::String; headers::Dict{String,String}=Dict{String,String}(), get_return_type::Function=(default,data)->default, @@ -108,7 +118,7 @@ In addition to these standard Julia methods, these convenience methods are also E.g: -``` +```julia # access o.field.subfield1.subfield2 if haspropertyat(o, "field", "subfield1", "subfield2") getpropertyat(o, "field", "subfield1", "subfield2") diff --git a/plugin/build.sh b/plugin/build.sh index 9e2f504..4aae90c 100755 --- a/plugin/build.sh +++ b/plugin/build.sh @@ -5,7 +5,7 @@ cd ${DIR}/../plugin echo "Building Julia plugin..." mvn package mvn dependency:resolve dependency:build-classpath -Dmdep.outputFile=classpath.tmp -echo "`cat classpath.tmp`:$DIR/target/julia-swagger-codegen-0.0.3.jar" > classpath +echo "`cat classpath.tmp`:$DIR/target/julia-swagger-codegen-0.0.4.jar" > classpath rm -f ./classpath.tmp echo "Build successful" echo "---------------------------------------" diff --git a/plugin/pom.xml b/plugin/pom.xml index 32de308..56b2b2a 100644 --- a/plugin/pom.xml +++ b/plugin/pom.xml @@ -4,7 +4,7 @@ com.juliacomputing.swagger.codegen julia-swagger-codegen jar - 0.0.3 + 0.0.4 julia-swagger-codegen http://maven.apache.org diff --git a/plugin/src/main/resources/julia/api.mustache b/plugin/src/main/resources/julia/api.mustache index e850990..9bec1e6 100644 --- a/plugin/src/main/resources/julia/api.mustache +++ b/plugin/src/main/resources/julia/api.mustache @@ -13,7 +13,7 @@ Param: {{paramName}}::{{dataType}}{{#required}} (required){{/required}} {{/allParams}} Return: {{#returnType}}{{returnType}}{{/returnType}}{{^returnType}}Nothing{{/returnType}} """ -function {{operationId}}(_api::{{classname}}{{#allParams}}{{#required}}, {{paramName}}{{^isBodyParam}}::{{dataType}}{{/isBodyParam}}{{/required}}{{/allParams}};{{#allParams}}{{^required}} {{paramName}}=nothing,{{/required}}{{/allParams}} _mediaType=nothing) +function _swaggerinternal_{{operationId}}(_api::{{classname}}{{#allParams}}{{#required}}, {{paramName}}{{^isBodyParam}}::{{dataType}}{{/isBodyParam}}{{/required}}{{/allParams}};{{#allParams}}{{^required}} {{paramName}}=nothing,{{/required}}{{/allParams}} _mediaType=nothing) {{#allParams}} {{#hasValidation}} {{#maxLength}} @@ -57,9 +57,19 @@ function {{operationId}}(_api::{{classname}}{{#allParams}}{{#required}}, {{param {{/formParams}} Swagger.set_header_accept(_ctx, [{{#produces}}"{{{mediaType}}}"{{#hasMore}}, {{/hasMore}}{{/produces}}]) Swagger.set_header_content_type(_ctx, (_mediaType === nothing) ? [{{#consumes}}"{{{mediaType}}}"{{#hasMore}}, {{/hasMore}}{{/consumes}}] : [_mediaType]) + return _ctx +end + +function {{operationId}}(_api::{{classname}}{{#allParams}}{{#required}}, {{paramName}}{{^isBodyParam}}::{{dataType}}{{/isBodyParam}}{{/required}}{{/allParams}};{{#allParams}}{{^required}} {{paramName}}=nothing,{{/required}}{{/allParams}} _mediaType=nothing) + _ctx = _swaggerinternal_{{operationId}}(_api{{#allParams}}{{#required}}, {{paramName}}{{/required}}{{/allParams}};{{#allParams}}{{^required}} {{paramName}}={{paramName}},{{/required}}{{/allParams}} _mediaType=_mediaType) Swagger.exec(_ctx) end +function {{operationId}}(_api::{{classname}}, response_stream::Channel{{#allParams}}{{#required}}, {{paramName}}{{^isBodyParam}}::{{dataType}}{{/isBodyParam}}{{/required}}{{/allParams}};{{#allParams}}{{^required}} {{paramName}}=nothing,{{/required}}{{/allParams}} _mediaType=nothing) + _ctx = _swaggerinternal_{{operationId}}(_api{{#allParams}}{{#required}}, {{paramName}}{{/required}}{{/allParams}};{{#allParams}}{{^required}} {{paramName}}={{paramName}},{{/required}}{{/allParams}} _mediaType=_mediaType) + Swagger.exec(_ctx, response_stream) +end + {{/operation}} export {{#operation}}{{operationId}}{{#hasMore}}, {{/hasMore}}{{/operation}} {{/operations}} diff --git a/src/client.jl b/src/client.jl index f5da3ac..96c259f 100644 --- a/src/client.jl +++ b/src/client.jl @@ -73,8 +73,8 @@ struct Client end end -set_user_agent(client::Client, ua::String) = set_header("User-Agent", ua) -set_cookie(client::Client, ck::String) = set_header("Cookie", ck) +set_user_agent(client::Client, ua::String) = set_header(client, "User-Agent", ua) +set_cookie(client::Client, ck::String) = set_header(client, "Cookie", ck) set_header(client::Client, name::String, value::String) = (client.headers[name] = value) struct Ctx @@ -180,13 +180,13 @@ function prep_args(ctx::Ctx) return body, kwargs end -response(::Type{Nothing}, resp::HTTP.Response) = nothing::Nothing -response(::Type{T}, resp::HTTP.Response) where {T <: Real} = response(T, resp.body)::T -response(::Type{T}, resp::HTTP.Response) where {T <: String} = response(T, resp.body)::T -function response(::Type{T}, resp::HTTP.Response) where {T} +response(::Type{Nothing}, resp::HTTP.Response, body=resp.body) = nothing::Nothing +response(::Type{T}, resp::HTTP.Response, body=resp.body) where {T <: Real} = response(T, body)::T +response(::Type{T}, resp::HTTP.Response, body=resp.body) where {T <: String} = response(T, body)::T +function response(::Type{T}, resp::HTTP.Response, body=resp.body) where {T} ctype = HTTP.header(resp, "Content-Type", "application/json") - (length(resp.body) == 0) && return T() - v = response(T, is_json_mime(ctype) ? JSON.parse(String(resp.body)) : resp.body) + (length(body) == 0) && return T() + v = response(T, is_json_mime(ctype) ? JSON.parse(String(body)) : body) v::T end response(::Type{T}, data::Vector{UInt8}) where {T<:Real} = parse(T, String(data)) @@ -197,7 +197,38 @@ response(::Type{T}, data::Dict{String,Any}) where {T} = from_json(T, data)::T response(::Type{T}, data::Dict{String,Any}) where {T<:Dict} = convert(T, data) response(::Type{Vector{T}}, data::Vector{V}) where {T,V} = [response(T, v) for v in data] -function exec(ctx::Ctx) +struct ChunkReader + http + buffered_input::Base.BufferStream + buffer_task::Task + + function ChunkReader(http) + buffered_input = Base.BufferStream() + buffer_task = @async try + write(buffered_input, http) + close(buffered_input) + catch ex + @error("exception reading http stream", exception=(ex, catch_backtrace())) + end + new(http, buffered_input, buffer_task) + end +end + +function Base.iterate(iter::ChunkReader, _state=nothing) + if eof(iter.buffered_input) + return nothing + else + out = IOBuffer() + while !eof(iter.buffered_input) + byte = read(iter.buffered_input, UInt8) + (byte == codepoint('\n')) && break + write(out, byte) + end + return (take!(out), iter) + end +end + +function do_request(ctx::Ctx, stream::Bool=false; stream_to::Union{Channel,Nothing}=nothing) resource_path = replace(ctx.resource, "{format}"=>"json") for (k,v) in ctx.path resource_path = replace(resource_path, "{$k}"=>v) @@ -205,14 +236,45 @@ function exec(ctx::Ctx) # TODO: use auth_settings for authentication body, kwargs = prep_args(ctx) - if body !== nothing - resp = HTTP.request(uppercase(ctx.method), HTTP.URIs.URI(resource_path), ctx.header, body; kwargs...) + if stream + @assert stream_to !== nothing + end + + resp = nothing + + function process_stream(http) + if body !== nothing + write(http, body) + end + resp = startread(http) + for chunk in ChunkReader(http) + return_type = ctx.client.get_return_type(ctx.return_type, String(copy(chunk))) + data = response(return_type, resp, chunk) + put!(stream_to, data) + end + end + + if stream + HTTP.open(process_stream, uppercase(ctx.method), HTTP.URIs.URI(resource_path), ctx.header; kwargs...) + close(stream_to) else - resp = HTTP.request(uppercase(ctx.method), HTTP.URIs.URI(resource_path), ctx.header; kwargs...) + if body !== nothing + resp = HTTP.request(uppercase(ctx.method), HTTP.URIs.URI(resource_path), ctx.header, body; kwargs...) + else + resp = HTTP.request(uppercase(ctx.method), HTTP.URIs.URI(resource_path), ctx.header; kwargs...) + end end + + return resp +end + +function exec(ctx::Ctx, stream_to::Union{Channel,Nothing}=nothing) + stream = stream_to !== nothing + resp = do_request(ctx, stream; stream_to=stream_to) + (200 <= resp.status <= 206) || throw(ApiException(resp)) - response(ctx.client.get_return_type(ctx.return_type, resp), resp) + return stream ? resp : response(ctx.client.get_return_type(ctx.return_type, resp), resp) end property_type(::Type{T}, name::Symbol) where {T<:SwaggerModel} = error("invalid type $T") diff --git a/test/petstore/test_StoreApi.jl b/test/petstore/test_StoreApi.jl index 65f075f..c92f66d 100644 --- a/test/petstore/test_StoreApi.jl +++ b/test/petstore/test_StoreApi.jl @@ -27,6 +27,21 @@ function test(uri) @test isa(order, Order) @test order.id == 10 + println(" - getOrderById (async)") + response_channel = Channel{Order}(1) + @test_throws Swagger.ValidationException getOrderById(api, response_channel, 0) + @sync begin + @async begin + resp = getOrderById(api, response_channel, 10) + @test (200 <= resp.status <= 206) + end + @async begin + order = take!(response_channel) + @test isa(order, Order) + @test order.id == 10 + end + end + println(" - deleteOrder") @test deleteOrder(api, 10) == nothing