diff --git a/.gitignore b/.gitignore index 1abdfef..7b2f934 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,4 @@ delete-me # generated files *.rockspec src/resty/aws/raw-api +.DS_Store diff --git a/README.md b/README.md index ab06a1d..3c553e7 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,10 @@ Release process: 1. upload using: `VERSION=x.y.z APIKEY=abc... make upload` 1. test installing the rock from LuaRocks +### 1.6.0 (04-Jun-2024) + +- feat: update the AWS SDK to v2.1353.0 + [117](https://github.com/Kong/lua-resty-aws/pull/117) ### 1.5.0 (20-May-2024) diff --git a/docs/classes/AWS.html b/docs/classes/AWS.html index 2b5adde..eae8f68 100644 --- a/docs/classes/AWS.html +++ b/docs/classes/AWS.html @@ -151,7 +151,7 @@

Usage:

generated by LDoc 1.5.0 -Last updated 2024-05-20 09:07:14 +Last updated 2024-06-04 09:33:12
diff --git a/docs/classes/ChainableTemporaryCredentials.html b/docs/classes/ChainableTemporaryCredentials.html index f7c2402..c80c648 100644 --- a/docs/classes/ChainableTemporaryCredentials.html +++ b/docs/classes/ChainableTemporaryCredentials.html @@ -147,7 +147,7 @@

Usage:

generated by LDoc 1.5.0 -Last updated 2024-05-20 09:07:14 +Last updated 2024-06-04 09:33:12
diff --git a/docs/classes/CredentialProviderChain.html b/docs/classes/CredentialProviderChain.html index 465b752..01936f0 100644 --- a/docs/classes/CredentialProviderChain.html +++ b/docs/classes/CredentialProviderChain.html @@ -131,7 +131,7 @@

Parameters:

generated by LDoc 1.5.0 -Last updated 2024-05-20 09:07:14 +Last updated 2024-06-04 09:33:12
diff --git a/docs/classes/Credentials.html b/docs/classes/Credentials.html index be4caf1..2dfb45b 100644 --- a/docs/classes/Credentials.html +++ b/docs/classes/Credentials.html @@ -266,7 +266,7 @@

Returns:

generated by LDoc 1.5.0 -Last updated 2024-05-20 09:07:14 +Last updated 2024-06-04 09:33:12
diff --git a/docs/classes/EC2MetadataCredentials.html b/docs/classes/EC2MetadataCredentials.html index 17f5467..3bfeb21 100644 --- a/docs/classes/EC2MetadataCredentials.html +++ b/docs/classes/EC2MetadataCredentials.html @@ -113,7 +113,7 @@

Parameters:

generated by LDoc 1.5.0 -Last updated 2024-05-20 09:07:14 +Last updated 2024-06-04 09:33:12
diff --git a/docs/classes/EnvironmentCredentials.html b/docs/classes/EnvironmentCredentials.html index 3b33f68..005588d 100644 --- a/docs/classes/EnvironmentCredentials.html +++ b/docs/classes/EnvironmentCredentials.html @@ -122,7 +122,7 @@

Parameters:

generated by LDoc 1.5.0 -Last updated 2024-05-20 09:07:14 +Last updated 2024-06-04 09:33:12
diff --git a/docs/classes/RemoteCredentials.html b/docs/classes/RemoteCredentials.html index 7ed784c..07f3ca3 100644 --- a/docs/classes/RemoteCredentials.html +++ b/docs/classes/RemoteCredentials.html @@ -113,7 +113,7 @@

Parameters:

generated by LDoc 1.5.0 -Last updated 2024-05-20 09:07:14 +Last updated 2024-06-04 09:33:12
diff --git a/docs/classes/SharedFileCredentials.html b/docs/classes/SharedFileCredentials.html index 92c7081..6b93513 100644 --- a/docs/classes/SharedFileCredentials.html +++ b/docs/classes/SharedFileCredentials.html @@ -113,7 +113,7 @@

Parameters:

generated by LDoc 1.5.0 -Last updated 2024-05-20 09:07:14 +Last updated 2024-06-04 09:33:12
diff --git a/docs/classes/TokenFileWebIdentityCredentials.html b/docs/classes/TokenFileWebIdentityCredentials.html index 9ab0caa..b7a62da 100644 --- a/docs/classes/TokenFileWebIdentityCredentials.html +++ b/docs/classes/TokenFileWebIdentityCredentials.html @@ -128,7 +128,7 @@

Parameters:

generated by LDoc 1.5.0 -Last updated 2024-05-20 09:07:14 +Last updated 2024-06-04 09:33:12
diff --git a/docs/index.html b/docs/index.html index 904c40d..e23b82e 100644 --- a/docs/index.html +++ b/docs/index.html @@ -126,7 +126,7 @@

Topics

generated by LDoc 1.5.0 -Last updated 2024-05-20 09:07:14 +Last updated 2024-06-04 09:33:12
diff --git a/docs/modules/resty.aws.config.html b/docs/modules/resty.aws.config.html index 114d244..b6d7c5f 100644 --- a/docs/modules/resty.aws.config.html +++ b/docs/modules/resty.aws.config.html @@ -356,7 +356,7 @@

Returns:

generated by LDoc 1.5.0 -Last updated 2024-05-20 09:07:14 +Last updated 2024-06-04 09:33:12
diff --git a/docs/modules/resty.aws.service.rds.signer.html b/docs/modules/resty.aws.service.rds.signer.html index e608282..1864e75 100644 --- a/docs/modules/resty.aws.service.rds.signer.html +++ b/docs/modules/resty.aws.service.rds.signer.html @@ -191,7 +191,7 @@

Usage:

generated by LDoc 1.5.0 -Last updated 2024-05-20 09:07:14 +Last updated 2024-06-04 09:33:12
diff --git a/docs/modules/resty.aws.utils.html b/docs/modules/resty.aws.utils.html index b43659f..a06fd59 100644 --- a/docs/modules/resty.aws.utils.html +++ b/docs/modules/resty.aws.utils.html @@ -196,7 +196,7 @@

Returns:

generated by LDoc 1.5.0 -Last updated 2024-05-20 09:07:14 +Last updated 2024-06-04 09:33:12
diff --git a/docs/topics/README.md.html b/docs/topics/README.md.html index 43e079d..5bc6524 100644 --- a/docs/topics/README.md.html +++ b/docs/topics/README.md.html @@ -289,6 +289,12 @@

History

  • test installing the rock from LuaRocks
    • +

    1.6.0 (04-Jun-2024)

    + +

    1.5.0 (20-May-2024)

    @@ -563,7 +569,7 @@

    0.1 (03-Feb-2021) Initial released version

    generated by LDoc 1.5.0 -Last updated 2024-05-20 09:07:14 +Last updated 2024-06-04 09:33:12
    diff --git a/spec/01-generic/04-stream-decoder_spec.lua b/spec/01-generic/04-stream-decoder_spec.lua new file mode 100644 index 0000000..e7f7264 --- /dev/null +++ b/spec/01-generic/04-stream-decoder_spec.lua @@ -0,0 +1,124 @@ +local tablex = require "pl.tablex" + +describe("stream decoder", function() + setup(function() + AWS = require "resty.aws" + end) + + teardown(function() + AWS = nil + package.loaded["resty.aws"] = nil + end) + + + it("decodes multiple messages from a single chunk", function() + local chunk_in_hex = "000000760000005296d5fade0b3a6576656e742d7479706507000c6d65737361676553746172740d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22726f6c65223a22617373697374616e74227d6fa8c599000001110000005706f176a90b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2250692028cf80292069732061206d617468656d61746963616c20636f6e7374616e74207468617420726570726573656e7473207468652063697263756d666572656e63652d746f2d6469616d6574657220726174696f206f66206120636972636c652e204974277320617070726f78696d6174656c7920332e31343135392e227d7d7d4e31940000007d00000056e6680fd60b3a6576656e742d74797065070010636f6e74656e74426c6f636b53746f700d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a307db32e4d340000007a00000051ca2c46650b3a6576656e742d7479706507000b6d65737361676553746f700d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b2273746f70526561736f6e223a22656e645f7475726e227d5fbf09fc000000b90000004ee991d99b0b3a6576656e742d747970650700086d657461646174610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226d657472696373223a7b226c6174656e63794d73223a313530367d2c227573616765223a7b22696e707574546f6b656e73223a382c226f7574707574546f6b656e73223a33392c22746f74616c546f6b656e73223a34377d7d5ad2dd7b" + local _STREAM = require("resty.aws.stream") + local parser, err = _STREAM:new(chunk_in_hex, true) + + if err then + assert.equal(err, nil) + return + end + + local messages = {} + + while true do + local msg = parser:next_message() + + if not msg then + break + end + + messages[#messages+1] = msg + end + + assert.same(messages, { + { + body = "{\"role\":\"assistant\"}", + headers = { + { + key = ":event-type", + value = "messageStart" + }, + { + key = ":content-type", + value = "application/json" + }, + { + key = ":message-type", + value = "event" + } + } + }, + { + body = "{\"contentBlockIndex\":0,\"delta\":{\"text\":\"Pi (π) is a mathematical constant that represents the circumference-to-diameter ratio of a circle. It's approximately 3.14159.\"}}", + headers = { + { + key = ":event-type", + value = "contentBlockDelta" + }, + { + key = ":content-type", + value = "application/json" + }, + { + key = ":message-type", + value = "event" + } + } + }, + { + body = "{\"contentBlockIndex\":0}", + headers = { + { + key = ":event-type", + value = "contentBlockStop" + }, + { + key = ":content-type", + value = "application/json" + }, + { + key = ":message-type", + value = "event" + } + } + }, + { + body = "{\"stopReason\":\"end_turn\"}", + headers = { + { + key = ":event-type", + value = "messageStop" + }, + { + key = ":content-type", + value = "application/json" + }, + { + key = ":message-type", + value = "event" + } + } + }, + { + body = "{\"metrics\":{\"latencyMs\":1506},\"usage\":{\"inputTokens\":8,\"outputTokens\":39,\"totalTokens\":47}}", + headers = { + { + key = ":event-type", + value = "metadata" + }, + { + key = ":content-type", + value = "application/json" + }, + { + key = ":message-type", + value = "event" + } + } + } + }) + end) +end) diff --git a/src/resty/aws/request/execute.lua b/src/resty/aws/request/execute.lua index 88ac2d7..02236d6 100644 --- a/src/resty/aws/request/execute.lua +++ b/src/resty/aws/request/execute.lua @@ -50,15 +50,22 @@ local function execute_request(signed_request) tostring(err)) end + local body, body_reader - local body do - if response.has_body then - body, err = response:read_body() - if not body then - return nil, ("failed reading response body from '%s:%s': %s"):format( - tostring(signed_request.host), - tostring(signed_request.port), - tostring(err)) + if response.headers["application/vnd.amazon.eventstream"] then + body_reader = response.body_reader + else + local this_body do + if response.has_body then + this_body, err = response:read_body() + if not this_body then + return nil, ("failed reading response body from '%s:%s': %s"):format( + tostring(signed_request.host), + tostring(signed_request.port), + tostring(err)) + end + + body = this_body end end end @@ -82,7 +89,8 @@ local function execute_request(signed_request) status = response.status, reason = response.reason, headers = response.headers, - body = body + body = body, + body_reader = body_reader, } end diff --git a/src/resty/aws/stream/init.lua b/src/resty/aws/stream/init.lua new file mode 100644 index 0000000..1732456 --- /dev/null +++ b/src/resty/aws/stream/init.lua @@ -0,0 +1,189 @@ +--- Stream class. +-- Decodes AWS response-stream types, currently application/vnd.amazon.eventstream +-- @classmod Stream + +local buf = require("string.buffer") +local to_hex = require("resty.string").to_hex + +local Stream = {} +Stream.__index = Stream + + +local _HEADER_EXTRACTORS = { + -- bool true + [0] = function(stream) + return true, 0 + end, + + -- bool false + [1] = function(stream) + return false, 0 + end, + + -- string type + [7] = function(stream) + local header_value_len = stream:next_int(16) + return stream:next_utf_8(header_value_len), header_value_len + 2 -- add the 2 bits read for the length + end, + + -- TODO ADD THE REST OF THE DATA TYPES + -- EVEN THOUGH THEY'RE NOT REALLY USED +} + +--- Constructor. +-- @function aws:Stream +-- @param chunk string complete AWS response stream chunk for decoding +-- @param is_hex boolean specify if the chunk bytes are already decoded to hex +-- @usage +-- local stream_parser = stream:new("00000120af0310f.......", true) +-- local next, err = stream_parser:next_message() +function Stream:new(chunk, is_hex) + local self = {} -- override 'self' to be the new object/class + setmetatable(self, Stream) + + if #chunk < ((is_hex and 32) or 16) then + return nil, "cannot parse a chunk less than 16 bytes long" + end + + self.read_count = 0 + self.chunk = buf.new() + self.chunk:put((is_hex and chunk) or to_hex(chunk)) + + return self +end + + +--- return the next `count` ascii bytes from the front of the chunk +--- and then trims the chunk of those bytes +-- @param count number whole utf-8 bytes to return +-- @return string resulting utf-8 string +function Stream:next_utf_8(count) + local utf_bytes = self:next_bytes(count) + + local ascii_string = "" + for i = 1, #utf_bytes, 2 do + local hex_byte = utf_bytes:sub(i, i + 1) + local ascii_byte = string.char(tonumber(hex_byte, 16)) + ascii_string = ascii_string .. ascii_byte + end + return ascii_string +end + +--- returns the next `count` bytes from the front of the chunk +--- and then trims the chunk of those bytes +-- @param count number whole integer of bytes to return +-- @return string hex-encoded next `count` bytes +function Stream:next_bytes(count) + if not self.chunk then + return nil, "function cannot be called on its own - initialise a chunk reader with :new(chunk)" + end + + local bytes = self.chunk:get(count * 2) + self.read_count = (count) + self.read_count + + return bytes +end + +--- returns the next unsigned int from the front of the chunk +--- and then trims the chunk of those bytes +-- @param size integer bit length (8, 16, 32, etc) +-- @return number whole integer of size specified +-- @return string the original bytes, for reference/checksums +function Stream:next_int(size) + if not self.chunk then + return nil, nil, "function cannot be called on its own - initialise a chunk reader with :new(chunk)" + end + + if size < 8 then + return nil, nil, "cannot work on integers smaller than 8 bits long" + end + + local int, err = self:next_bytes(size / 8, trim) + if err then + return nil, nil, err + end + + return tonumber(int, 16), int +end + +--- returns the next message in the chunk, as a table. +--- can be used as an iterator. +-- @return table formatted next message from the given constructor chunk +function Stream:next_message() + if not self.chunk then + return nil, "function cannot be called on its own - initialise a chunk reader with :new(chunk)" + end + + if #self.chunk < 1 then + return false + end + + -- get the message length and pull that many bytes + -- + -- this is a chicken and egg problem, because we need to + -- read the message to get the length, to then re-read the + -- whole message at correct offset + local msg_len, orig_len, err = self:next_int(32) + if err then + return err + end + + -- get the headers length + local headers_len, orig_headers_len, err = self:next_int(32) + + -- get the preamble checksum + local preamble_checksum, orig_preamble_checksum, err = self:next_int(32) + + -- TODO: calculate checksum + -- local result = crc32(orig_len .. origin_headers_len, preamble_checksum) + -- if not result then + -- return nil, "preamble checksum failed - message is corrupted" + -- end + + -- pull the headers from the buf + local headers = {} + local headers_bytes_read = 0 + + while headers_bytes_read < headers_len do + -- the next 8-bit int is the "header key length" + local header_key_len = self:next_int(8) + local header_key = self:next_utf_8(header_key_len) + headers_bytes_read = 1 + header_key_len + headers_bytes_read + + -- next 8-bits is the header type, which is an enum + local header_type = self:next_int(8) + headers_bytes_read = 1 + headers_bytes_read + + -- depending on the header type, depends on how long the header should max out at + local header_value, header_value_len = _HEADER_EXTRACTORS[header_type](self) + headers_bytes_read = header_value_len + headers_bytes_read + + headers[#headers+1] = { + key = header_key, + value = header_value, + } + end + + -- finally, extract the body as a string by + -- subtracting what's read so far from the + -- total length obtained right at the start + local body = self:next_utf_8(msg_len - self.read_count - 4) + + -- last 4 bytes is a body checksum + local msg_checksum = self:next_int(32) + -- TODO CHECK FULL MESSAGE CHECKSUM + -- local result = crc32(original_full_msg, msg_checksum) + -- if not result then + -- return nil, "preamble checksum failed - message is corrupted" + -- end + + -- rewind the tape + self.read_count = 0 + + return { + headers = headers, + body = body, + } +end + +return Stream diff --git a/test-chunk.lua b/test-chunk.lua new file mode 100644 index 0000000..45d9891 --- /dev/null +++ b/test-chunk.lua @@ -0,0 +1,30 @@ +setmetatable(_G, nil) -- disable global warnings + +-- make sure we can use dev code +package.path = "./src/?.lua;./src/?/init.lua;"..package.path + +-- quick debug dump function +local dump = function(...) + local t = { n = select("#", ...), ...} + if t.n == 1 and type(t[1]) == "table" then t = t[1] end + print(require("pl.pretty").write(t)) +end + +local chunk_in_hex = "000000760000005296d5fade0b3a6576656e742d7479706507000c6d65737361676553746172740d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22726f6c65223a22617373697374616e74227d6fa8c599000001110000005706f176a90b3a6576656e742d74797065070011636f6e74656e74426c6f636b44656c74610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a302c2264656c7461223a7b2274657874223a2250692028cf80292069732061206d617468656d61746963616c20636f6e7374616e74207468617420726570726573656e7473207468652063697263756d666572656e63652d746f2d6469616d6574657220726174696f206f66206120636972636c652e204974277320617070726f78696d6174656c7920332e31343135392e227d7d7d4e31940000007d00000056e6680fd60b3a6576656e742d74797065070010636f6e74656e74426c6f636b53746f700d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b22636f6e74656e74426c6f636b496e646578223a307db32e4d340000007a00000051ca2c46650b3a6576656e742d7479706507000b6d65737361676553746f700d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b2273746f70526561736f6e223a22656e645f7475726e227d5fbf09fc000000b90000004ee991d99b0b3a6576656e742d747970650700086d657461646174610d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226d657472696373223a7b226c6174656e63794d73223a313530367d2c227573616765223a7b22696e707574546f6b656e73223a382c226f7574707574546f6b656e73223a33392c22746f74616c546f6b656e73223a34377d7d5ad2dd7b" +local _STREAM = require("resty.aws.stream") +local parser, err = _STREAM:new(chunk_in_hex, true) + +if err then + print("ERROR: ", err) + return +end + +while true do + local msg = parser:next_message() + + if not msg then + break + end + + print(dump(msg)) +end diff --git a/update_api_files.sh b/update_api_files.sh index 93a531a..35933ee 100755 --- a/update_api_files.sh +++ b/update_api_files.sh @@ -6,7 +6,7 @@ # It will convert the service descriptions of the specified SDK version # (see SDK_VERSION_TAG) into Lua modules and generate a rockspec. -SDK_VERSION_TAG=v2.751.0 +SDK_VERSION_TAG=v2.1353.0 # ----------- nothing to customize below ----------- TARGET=./src/resty/aws/raw-api