Skip to content
This repository has been archived by the owner on Oct 18, 2023. It is now read-only.

Commit

Permalink
fixing the tests with fixtures
Browse files Browse the repository at this point in the history
  • Loading branch information
vggonzal authored and vggonzal committed Aug 18, 2023
1 parent 5728628 commit d4d2985
Show file tree
Hide file tree
Showing 123 changed files with 1,527 additions and 106 deletions.
9 changes: 1 addition & 8 deletions .github/workflows/build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,15 +167,8 @@ jobs:
poetry build
#- name: Setup local DynamoDB
# run: docker run -d -p 8000:8000 amazon/dynamodb-local

#- name: Load sample
# run: python3 tests/example_load_data.py

#- name: Test with pytest
# run: pytest tests/test_api.py
# run: pytest



Expand Down
12 changes: 12 additions & 0 deletions conftest.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
'''
conftest file to set up local dynamodb connection
'''
from pytest_dynamodb import factories
from tests.database_fixtures import data_table
from tests.database_fixtures import data_table_with_transactions

dynamo_test_proc = factories.dynamodb_proc(
dynamodb_dir="tests/dynamodb_local",
port=8001)

dynamo_db_resource = factories.dynamodb("dynamo_test_proc")
2 changes: 1 addition & 1 deletion hydrocronapi/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ def main():
app.add_api('swagger.yaml',
arguments={'title': 'Get time series data from SWOT observations for reaches, nodes, and/or lakes'},
pythonic_params=True)
app.run(port=8080)
app.run(port=8081)


if __name__ == '__main__':
Expand Down
12 changes: 6 additions & 6 deletions hydrocronapi/controllers/db/db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,11 @@
import logging
from datetime import datetime
from typing import Generator
import boto3.dynamodb.conditions as conditions

import boto3

dynamodb = boto3.client('dynamodb')
dynamodb = boto3.resource('dynamodb')

logger = logging.getLogger()

Expand Down Expand Up @@ -66,11 +67,10 @@ def get_reach_series_by_feature_id(feature_id: str, start_time: datetime, end_ti
:rtype: None
"""

response = dynamodb.get_item(
TableName='hydrocron_swot_reaches_test',
Key={
'reach_id': {'S': feature_id}
}
table = dynamodb.Table('hydrocron_swot_reaches_test')
response = table.query(
KeyConditionExpression= \
conditions.Key("PK").eq(feature_id)
)
return response

Expand Down
116 changes: 80 additions & 36 deletions hydrocronapi/controllers/db/hydrocron_database.py
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,34 @@
# pylint: disable=R1720
# pylint: disable=C0121
# pylint: disable=W0612
# pylint: disable=C0103
# pylint: disable=C0115
# pylint: disable=C0114
"""
Hydrocron Database module
"""
import logging
from botocore.exceptions import ClientError
from boto3.dynamodb.conditions import Key

logger = logging.getLogger(__name__)


class Hydrocron_DB:
class DynamoKeys:
"""
Represents the partition and sort keys for a dynamoDB table
"""
def __init__(
self,
partition_key,
partition_key_type,
sort_key,
sort_key_type):

self.partition_key = partition_key
self.partition_key_type = partition_key_type
self.sort_key = sort_key
self.sort_key_type = sort_key_type


class HydrocronDB:
"""
Hydrocron database class.
"""
def __init__(self, dyn_resource):
"""
Parameters
Expand Down Expand Up @@ -49,14 +66,16 @@ def table_exists(self, table_name):
logger.error(
"Couldn't check for existence of %s. %s: %s",
table_name,
err.response['Error']['Code'], err.response['Error']['Message'])
err.response['Error']['Code'],
err.response['Error']['Message'])
raise

return exists

def create_table(self, table_name, partition_key, partition_key_type, sort_key, sort_key_type):
def create_table(self, table_name, dynamo_keys):
"""
Creates an Amazon DynamoDB table to store SWOT River Reach, Node, or Lake data for the Hydrocron API.
Creates an Amazon DynamoDB table to store SWOT River Reach,
Node, or Lake data for the Hydrocron API.
Parameters
---------
Expand All @@ -69,17 +88,21 @@ def create_table(self, table_name, partition_key, partition_key_type, sort_key,
The newly created table.
"""
try:
new_table = Hydrocron_Table(self.dyn_resource,
table_name,
partition_key, partition_key_type,
sort_key, sort_key_type)
new_table = HydrocronTable(
self.dyn_resource,
table_name,
dynamo_keys.partition_key,
dynamo_keys.partition_key_type,
dynamo_keys.sort_key,
dynamo_keys.sort_key_type)

self.tables.append(new_table.table_name)

except ClientError as err:
logger.error(
"Couldn't create table %s. %s: %s", table_name,
err.response['Error']['Code'], err.response['Error']['Message'])
err.response['Error']['Code'],
err.response['Error']['Message'])
raise
else:
return new_table
Expand All @@ -105,7 +128,8 @@ def load_table(self, table_name):
except ClientError as err:
logger.error(
"Couldn't load table %s. %s: %s", table_name,
err.response['Error']['Code'], err.response['Error']['Message'])
err.response['Error']['Code'],
err.response['Error']['Message'])
raise
else:
return table
Expand All @@ -126,7 +150,8 @@ def list_tables(self):
except ClientError as err:
logger.error(
"Couldn't list tables. %s: %s",
err.response['Error']['Code'], err.response['Error']['Message'])
err.response['Error']['Code'],
err.response['Error']['Message'])
raise
else:
return self.tables
Expand All @@ -145,11 +170,15 @@ def delete_table(self, table_name):
except ClientError as err:
logger.error(
"Couldn't delete table. %s: %s",
err.response['Error']['Code'], err.response['Error']['Message'])
err.response['Error']['Code'],
err.response['Error']['Message'])
raise


class Hydrocron_Table:
class HydrocronTable:
"""
class docstring
"""
def __init__(self, dyn_resource,
table_name,
partition_key_name, partition_key_type,
Expand Down Expand Up @@ -183,8 +212,8 @@ def __init__(self, dyn_resource,

def create_table(self):
"""
Creates an Amazon DynamoDB table to store SWOT River Reach, Node, or Lake data for the Hydrocron API.
Creates an Amazon DynamoDB table to store SWOT River Reach,
Node, or Lake data for the Hydrocron API.
Returns
-------
Expand All @@ -195,19 +224,25 @@ def create_table(self):
self.table = self.dyn_resource.create_table(
TableName=self.table_name,
KeySchema=[
{'AttributeName': self.partition_key_name, 'KeyType': 'HASH'}, # Partition key
{'AttributeName': self.sort_key_name, 'KeyType': 'RANGE'} # Sort key
{'AttributeName': self.partition_key_name,
'KeyType': 'HASH'}, # Partition key
{'AttributeName': self.sort_key_name,
'KeyType': 'RANGE'} # Sort key
],
AttributeDefinitions=[
{'AttributeName': self.partition_key_name, 'AttributeType': self.partition_key_type},
{'AttributeName': self.sort_key_name, 'AttributeType': self.sort_key_type}
{'AttributeName': self.partition_key_name,
'AttributeType': self.partition_key_type},
{'AttributeName': self.sort_key_name,
'AttributeType': self.sort_key_type}
],
ProvisionedThroughput={'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10})
ProvisionedThroughput={'ReadCapacityUnits': 10,
'WriteCapacityUnits': 10})
self.table.wait_until_exists()
except ClientError as err:
logger.error(
"Couldn't create table %s. %s: %s", self.table_name,
err.response['Error']['Code'], err.response['Error']['Message'])
err.response['Error']['Code'],
err.response['Error']['Message'])
raise
else:
return self.table
Expand All @@ -218,7 +253,7 @@ def add_data(self, **kwargs):
Parameters
---------
**kwargs: All attributes to add to the item. Must include partition and sort keys
**kwargs: All attributes to add to the item. Must include partition and sort keys # noqa
"""

item_dict = {}
Expand All @@ -234,7 +269,8 @@ def add_data(self, **kwargs):
logger.error(
"Couldn't add item %s to table %s. Here's why: %s: %s",
self.partition_key_name, self.table.name,
err.response['Error']['Code'], err.response['Error']['Message'])
err.response['Error']['Code'],
err.response['Error']['Message'])
raise

def run_query(self, partition_key, sort_key=None):
Expand All @@ -258,24 +294,30 @@ def run_query(self, partition_key, sort_key=None):

try:
response = self.table.query(
KeyConditionExpression=(Key(self.partition_key_name).eq(partition_key)))
KeyConditionExpression=(
Key(self.partition_key_name).eq(partition_key)
))
except ClientError as err:
logger.error(
"Couldn't query for items: %s: %s",
err.response['Error']['Code'], err.response['Error']['Message'])
err.response['Error']['Code'],
err.response['Error']['Message'])
raise
else:
return response['Items']

else:
try:
response = self.table.query(
KeyConditionExpression=(Key(self.partition_key_name).eq(partition_key) &
Key(self.sort_key_name).eq(sort_key)))
KeyConditionExpression=(
Key(self.partition_key_name).eq(partition_key) &
Key(self.sort_key_name).eq(sort_key)
))
except ClientError as err:
logger.error(
"Couldn't query for items: %s: %s",
err.response['Error']['Code'], err.response['Error']['Message'])
err.response['Error']['Code'],
err.response['Error']['Message'])
raise
else:
return response['Items']
Expand All @@ -292,9 +334,11 @@ def delete_item(self, partition_key, sort_key):
The timestamp of the item to delete.
"""
try:
self.table.delete_item(Key={self.partition_key_name: partition_key, self.sort_key_name: sort_key})
self.table.delete_item(Key={self.partition_key_name: partition_key,
self.sort_key_name: sort_key})
except ClientError as err:
logger.error(
"Couldn't delete item %s. %s: %s", partition_key,
err.response['Error']['Code'], err.response['Error']['Message'])
err.response['Error']['Code'],
err.response['Error']['Message'])
raise
6 changes: 3 additions & 3 deletions hydrocronapi/controllers/db/db_local.py → hydrocronapi/controllers/db/no_db_local.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from typing import Generator

import boto3
from .hydrocron_database import Hydrocron_DB
from .hydrocron_database import HydrocronDB
from boto3.dynamodb.conditions import Key


Expand All @@ -29,9 +29,9 @@
aws_session_token='fake_session_token',
region_name='us-west-2')

dyndb_resource = session.resource('dynamodb', endpoint_url='http://localhost:8000')
dyndb_resource = session.resource('dynamodb', endpoint_url='http://localhost:8002')

dynamo_instance = Hydrocron_DB(dyn_resource=dyndb_resource)
dynamo_instance = HydrocronDB(dyn_resource=dyndb_resource)


def get_reach_series(start_time: datetime, end_time: datetime) -> Generator:
Expand Down
Loading

0 comments on commit d4d2985

Please sign in to comment.