Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Docstore/awsdynamodb: Supporting the AWS V2 SDK #3519

Open
wants to merge 27 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
235bb36
updating codec tests for v2 decoder types
Maldris Jan 28, 2025
9fe3f3f
rewriting aws dynamodb codec to use the types from v2 aws cdk
Maldris Jan 28, 2025
b3e6144
updating modules based on update progress
Maldris Jan 29, 2025
303bb45
re-introducing codecTester
Maldris Jan 29, 2025
eda4be6
updating query planner and builder behaviors with aws sdk v2 types
Maldris Jan 29, 2025
30a4f31
updating url opener parsing to use aws sdk v2 initialization helpers
Maldris Jan 29, 2025
084cda2
converting main dynamodb docstore logic to use aws sdk v2
Maldris Jan 29, 2025
722d6ce
updating example and dynamodb docstore benchmark and example test ref…
Maldris Jan 29, 2025
6717d84
updating comment to point to v2 documentation of aws session initiali…
Maldris Jan 30, 2025
af13aa0
removing commented out v1 code used as reference
Maldris Jan 30, 2025
59ae378
explicitly handle document encoding error (should never happen)
Maldris Jan 30, 2025
ff01d20
docstore/awsdynamodb's encodeDocKeyFields always returns map attribut…
Maldris Jan 30, 2025
984e123
reviewed todos against v1 behavior and added explanatory comments
Maldris Jan 30, 2025
c81935c
grouping encoder tests by type
Maldris Jan 30, 2025
eb9d4fb
adding tests for boolean false, multi-char and space containing string
Maldris Jan 30, 2025
6686fbd
centralizing attribute value ignore unexported definition
Maldris Jan 30, 2025
e1af8c3
adding decoder tests
Maldris Jan 30, 2025
d760c77
Merge branch 'master' into aws/v2/docstore
Maldris Jan 30, 2025
195ed3a
fixing removed import label from merge
Maldris Jan 30, 2025
f8f46fe
logging the test that fails for easier debug
Maldris Jan 30, 2025
9d56686
tests specify region, setup script does not, adding to make tests rep…
Maldris Jan 31, 2025
9de4a23
fixing error as to asset the correct type
Maldris Jan 31, 2025
3dbca27
updating error code lookup using new error as behavior from aws v2
Maldris Jan 31, 2025
b6fa9e3
fixing logic error in missing field key check
Maldris Jan 31, 2025
943f141
fixing different map behavior in aws dynamodb v2 not matching test
Maldris Jan 31, 2025
4539b08
updating error decoding to use correct type
Maldris Jan 31, 2025
4b5dd4b
clarifying errors with comments
Maldris Jan 31, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 28 additions & 40 deletions docstore/awsdynamodb/benchmark_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,15 @@
package awsdynamodb

import (
"context"
"fmt"
"net/http"
"strconv"
"testing"

"github.com/aws/aws-sdk-go/aws"
awscreds "github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/aws/aws-sdk-go/service/dynamodb/expression"
awsv2cfg "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
dyn2Types "github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
)

var benchmarkTableName = collectionName3
Expand All @@ -35,24 +34,24 @@ func BenchmarkPutVSTransact(b *testing.B) {
// The second way calls BatchGetItem followed by TransactWriteItem.
//
// The results show that separate PutItems are faster for up to two items.
sess, err := awsSession(region, http.DefaultClient)
cfg, err := awsv2cfg.LoadDefaultConfig(context.Background())
if err != nil {
b.Fatal(err)
b.Fatal("Error initializing aws session for benchmark: ", err)
}
db := dynamodb.New(sess)
db := dynamodb.NewFromConfig(cfg)

for nItems := 1; nItems <= 5; nItems++ {
b.Run(fmt.Sprintf("%d-Items", nItems), func(b *testing.B) {
var items []map[string]*dynamodb.AttributeValue
var items []map[string]dyn2Types.AttributeValue
for i := 0; i < nItems; i++ {
items = append(items, map[string]*dynamodb.AttributeValue{
"name": new(dynamodb.AttributeValue).SetS(fmt.Sprintf("pt-vs-transact-%d", i)),
"x": new(dynamodb.AttributeValue).SetN(strconv.Itoa(i)),
"rev": new(dynamodb.AttributeValue).SetN("1"),
items = append(items, map[string]dyn2Types.AttributeValue{
"name": &dyn2Types.AttributeValueMemberS{Value: fmt.Sprintf("pt-vs-transact-%d", i)},
"x": &dyn2Types.AttributeValueMemberN{Value: strconv.Itoa(i)},
"rev": &dyn2Types.AttributeValueMemberN{Value: "1"},
})
}
for _, item := range items {
_, err := db.PutItem(&dynamodb.PutItemInput{
_, err := db.PutItem(context.Background(), &dynamodb.PutItemInput{
TableName: &benchmarkTableName,
Item: item,
})
Expand All @@ -74,15 +73,15 @@ func BenchmarkPutVSTransact(b *testing.B) {
}
}

func putItems(b *testing.B, db *dynamodb.DynamoDB, items []map[string]*dynamodb.AttributeValue) {
func putItems(b *testing.B, db *dynamodb.Client, items []map[string]dyn2Types.AttributeValue) {
b.Helper()

for i, item := range items {
item["x"].SetN(strconv.Itoa(i + 1))
item["x"] = &dyn2Types.AttributeValueMemberN{Value: strconv.Itoa(i + 1)}
in := &dynamodb.PutItemInput{
TableName: &benchmarkTableName,
Item: item,
ReturnValues: aws.String("ALL_OLD"),
ReturnValues: dyn2Types.ReturnValueAllOld,
}
ce, err := expression.NewBuilder().
WithCondition(expression.Name("rev").Equal(expression.Value(1))).
Expand All @@ -93,7 +92,7 @@ func putItems(b *testing.B, db *dynamodb.DynamoDB, items []map[string]*dynamodb.
in.ExpressionAttributeNames = ce.Names()
in.ExpressionAttributeValues = ce.Values()
in.ConditionExpression = ce.Condition()
out, err := db.PutItem(in)
out, err := db.PutItem(context.Background(), in)
if err != nil {
b.Fatal(err)
}
Expand All @@ -103,15 +102,15 @@ func putItems(b *testing.B, db *dynamodb.DynamoDB, items []map[string]*dynamodb.
}
}

func batchGetTransactWrite(b *testing.B, db *dynamodb.DynamoDB, items []map[string]*dynamodb.AttributeValue) {
func batchGetTransactWrite(b *testing.B, db *dynamodb.Client, items []map[string]dyn2Types.AttributeValue) {
b.Helper()

keys := make([]map[string]*dynamodb.AttributeValue, len(items))
tws := make([]*dynamodb.TransactWriteItem, len(items))
keys := make([]map[string]dyn2Types.AttributeValue, len(items))
tws := make([]dyn2Types.TransactWriteItem, len(items))
for i, item := range items {
keys[i] = map[string]*dynamodb.AttributeValue{"name": items[i]["name"]}
item["x"].SetN(strconv.Itoa(i + 2))
put := &dynamodb.Put{TableName: &benchmarkTableName, Item: items[i]}
keys[i] = map[string]dyn2Types.AttributeValue{"name": items[i]["name"]}
item["x"] = &dyn2Types.AttributeValueMemberN{Value: strconv.Itoa(i + 2)}
put := &dyn2Types.Put{TableName: &benchmarkTableName, Item: items[i]}
ce, err := expression.NewBuilder().
WithCondition(expression.Name("rev").Equal(expression.Value(1))).
Build()
Expand All @@ -121,29 +120,18 @@ func batchGetTransactWrite(b *testing.B, db *dynamodb.DynamoDB, items []map[stri
put.ExpressionAttributeNames = ce.Names()
put.ExpressionAttributeValues = ce.Values()
put.ConditionExpression = ce.Condition()
tws[i] = &dynamodb.TransactWriteItem{Put: put}
tws[i] = dyn2Types.TransactWriteItem{Put: put}
}
_, err := db.BatchGetItem(&dynamodb.BatchGetItemInput{
RequestItems: map[string]*dynamodb.KeysAndAttributes{
_, err := db.BatchGetItem(context.Background(), &dynamodb.BatchGetItemInput{
RequestItems: map[string]dyn2Types.KeysAndAttributes{
benchmarkTableName: {Keys: keys},
},
})
if err != nil {
b.Fatal(err)
}
_, err = db.TransactWriteItems(&dynamodb.TransactWriteItemsInput{TransactItems: tws})
_, err = db.TransactWriteItems(context.Background(), &dynamodb.TransactWriteItemsInput{TransactItems: tws})
if err != nil {
b.Fatal(err)
}
}

func awsSession(region string, client *http.Client) (*session.Session, error) {
// Provide fake creds if running in replay mode.
var creds *awscreds.Credentials
return session.NewSession(&aws.Config{
HTTPClient: client,
Region: aws.String(region),
Credentials: creds,
MaxRetries: aws.Int(0),
})
}
Loading