From 04bec69844a5839e22cd9c6cb9aa11b7816d46a9 Mon Sep 17 00:00:00 2001 From: John Konderla Date: Sun, 11 Nov 2018 20:38:49 -0500 Subject: [PATCH 1/5] Added ability to update dots on PUT command --- Source/PutHotSpot.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/Source/PutHotSpot.py b/Source/PutHotSpot.py index b9eafcd..6ee3181 100644 --- a/Source/PutHotSpot.py +++ b/Source/PutHotSpot.py @@ -1,5 +1,5 @@ +import datetime import os -from datetime import datetime from decimal import Decimal import boto3 @@ -13,16 +13,34 @@ def put_hot_spot_handler(event, context): table = dynamodb.Table(os.getenv('TABLE_NAME')) lat = Decimal(str(event.get('lat', 35.7721))) lng = Decimal(str(event.get('lng', -78.6441))) - id = context.aws_request_id - hash = str(event.get('hash', 'no hash')) + location_id = event.get('locationID', '') + input_hash = str(event.get('hash', 'no hash')) print("lat, lng: {} {}".format(lat, lng)) - date_time = datetime.now() + date_time = datetime.datetime.now() color_code = event.get('colorCode', '1') key = '{:.1f}_{:.1f}'.format(lat, lng) print('key: {}'.format(key)) response = {} + if location_id: + print('looking for a location: {}'.format(location_id)) + two_hours_less = datetime.timedelta(hours=2) + minus_two = date_time - two_hours_less + query_response = table.query(IndexName='locationID-date_time-index', + KeyConditionExpression='locationID = :location_id and date_time > :date_time_value', + FilterExpression='lat_lng = :lat_lng_value', + ExpressionAttributeValues={ + ":location_id": location_id, + ":date_time_value": '{}'.format(minus_two), + ':lat_lng_value': key + }) + print('response.... {}'.format(query_response)) + if not query_response['Items']: + location_id = context.aws_request_id + else: + location_id = context.aws_request_id + try: response = table.put_item( Item={ @@ -30,8 +48,8 @@ def put_hot_spot_handler(event, context): 'lat': lat, 'lng': lng, 'date_time': "{}".format(date_time), - 'locationID': id, - 'hash': hash, + 'locationID': location_id, + 'hash': input_hash, 'colorCode': color_code } ) @@ -42,7 +60,7 @@ def put_hot_spot_handler(event, context): if response: return { "response": "success", - "locationID": id + "locationID": location_id } else: return {"message": "error"} @@ -51,15 +69,16 @@ def put_hot_spot_handler(event, context): if __name__ == '__main__': class Context: def __init__(self): - self.aws_request_id = '123' + self.aws_request_id = '123123123123' - os.environ['TABLE_NAME'] = "HotSpotAlpha" + os.environ['TABLE_NAME'] = "HotSpotDynamoDbTable-HotSpotTable-F6VB59AJ2YDK" event_dict = { 'lat': 123.2, 'lng': 321.1, - 'colorCode': 1 + 'colorCode': 1, + "locationID": '123123123123' } context = Context() From bfd02a97e18ae0d0142bf6aec11aee054410d433 Mon Sep 17 00:00:00 2001 From: John Konderla Date: Fri, 23 Nov 2018 11:29:52 -0500 Subject: [PATCH 2/5] Fixed some params for the CFTs Made better logging for the put lambda Added correct index permissions for the query for the put lambda --- Infrastructure/GetHotSpots.template | 8 ++++---- Infrastructure/HotSpotDynamoDbTable.template | 8 +++++++- Infrastructure/PutHotSpot.template | 17 +++++++++++++---- Source/PutHotSpot.py | 9 ++++++--- 4 files changed, 30 insertions(+), 12 deletions(-) diff --git a/Infrastructure/GetHotSpots.template b/Infrastructure/GetHotSpots.template index 7a8c530..3940299 100644 --- a/Infrastructure/GetHotSpots.template +++ b/Infrastructure/GetHotSpots.template @@ -8,10 +8,10 @@ Parameters: Type: String Description: The key of the zip file where the lambda code is stored in s3. Default: lambdas.zip - HotSpotDynamoDbTabeArn: + HotSpotDynamoDbTableArn: Type: String Description: The ARN of the DynamoDB table for HotSpot - HotSpotDynamoDbTabeName: + HotSpotDynamoDbTableName: Type: String Description: The name of the DynamoDB table for HotSpot Resources: @@ -26,7 +26,7 @@ Resources: Description: Get Hot Spots lambda code Environment: Variables: - TABLE_NAME: !Ref HotSpotDynamoDbTabeName + TABLE_NAME: !Ref HotSpotDynamoDbTableName Handler: GetHotSpots.get_hot_spots_handler MemorySize: 128 ReservedConcurrentExecutions: 1 @@ -60,7 +60,7 @@ Resources: - Effect: Allow Action: - dynamodb:Query - Resource: !Ref HotSpotDynamoDbTabeArn + Resource: !Ref HotSpotDynamoDbTableArn Outputs: diff --git a/Infrastructure/HotSpotDynamoDbTable.template b/Infrastructure/HotSpotDynamoDbTable.template index 811cc0e..3a2ea47 100644 --- a/Infrastructure/HotSpotDynamoDbTable.template +++ b/Infrastructure/HotSpotDynamoDbTable.template @@ -1,5 +1,11 @@ AWSTemplateFormatVersion: '2010-09-09' +Parameters: + HotSpotDynamoTableLocaionIndex: + Type: String + Description: Name of the Location ID index + Default: locationID-date_time-index + Resources: HotSpotTable: Type: AWS::DynamoDB::Table @@ -17,7 +23,7 @@ Resources: - AttributeName: date_time KeyType: RANGE GlobalSecondaryIndexes: - - IndexName: locationID-date_time-index + - IndexName: !Ref HotSpotDynamoTableLocaionIndex KeySchema: - AttributeName: locationID KeyType: HASH diff --git a/Infrastructure/PutHotSpot.template b/Infrastructure/PutHotSpot.template index 431417f..98b57c2 100644 --- a/Infrastructure/PutHotSpot.template +++ b/Infrastructure/PutHotSpot.template @@ -8,12 +8,16 @@ Parameters: Type: String Description: The key of the zip file where the lambda code is stored in s3. Default: lambdas.zip - HotSpotDynamoDbTabeArn: + HotSpotDynamoDbTalbeArn: Type: String Description: The ARN of the DynamoDB table for HotSpot - HotSpotDynamoDbTabeName: + HotSpotDynamoDbTableName: Type: String Description: The name of the DynamoDB table for HotSpot + HotSpotDynamoTableLocaionIndex: + Type: String + Description: The name for the index of the locationID in the HotSpot db + Default: locationID-date_time-index Resources: PutHotSpotLambda: Type: "AWS::Lambda::Function" @@ -26,7 +30,7 @@ Resources: Description: Put Hot Spots lambda code Environment: Variables: - TABLE_NAME: !Ref HotSpotDynamoDbTabeName + TABLE_NAME: !Ref HotSpotDynamoDbTableName Handler: PutHotSpot.put_hot_spot_handler MemorySize: 128 ReservedConcurrentExecutions: 1 @@ -61,7 +65,12 @@ Resources: Action: - dynamodb:PutItem - dynamodb:Query - Resource: !Ref HotSpotDynamoDbTabeArn + Resource: !Ref HotSpotDynamoDbTalbeArn + - Effect: Allow + Action: + - dynamodb:Query + Resource: !Sub ${HotSpotDynamoDbTalbeArn}/index/${HotSpotDynamoTableLocaionIndex} + Outputs: RoleArn: diff --git a/Source/PutHotSpot.py b/Source/PutHotSpot.py index 6ee3181..1251630 100644 --- a/Source/PutHotSpot.py +++ b/Source/PutHotSpot.py @@ -13,7 +13,7 @@ def put_hot_spot_handler(event, context): table = dynamodb.Table(os.getenv('TABLE_NAME')) lat = Decimal(str(event.get('lat', 35.7721))) lng = Decimal(str(event.get('lng', -78.6441))) - location_id = event.get('locationID', '') + location_id = str(event.get('locationID', '')) input_hash = str(event.get('hash', 'no hash')) print("lat, lng: {} {}".format(lat, lng)) date_time = datetime.datetime.now() @@ -58,12 +58,15 @@ def put_hot_spot_handler(event, context): print("error: {}".format(e.response['Error']['Message'])) if response: - return { + return_dict = { "response": "success", "locationID": location_id } else: - return {"message": "error"} + return_dict = {"message": "error"} + + print('return_dict: {}'.format(return_dict)) + return return_dict if __name__ == '__main__': From 6a7ea516448fdd4ef457901732c2228fb50b70c6 Mon Sep 17 00:00:00 2001 From: John Konderla Date: Fri, 23 Nov 2018 11:52:15 -0500 Subject: [PATCH 3/5] Removed the auto populate of no hash. No hash in request means no hash in the db. --- Source/PutHotSpot.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/Source/PutHotSpot.py b/Source/PutHotSpot.py index 1251630..ef5a589 100644 --- a/Source/PutHotSpot.py +++ b/Source/PutHotSpot.py @@ -14,10 +14,10 @@ def put_hot_spot_handler(event, context): lat = Decimal(str(event.get('lat', 35.7721))) lng = Decimal(str(event.get('lng', -78.6441))) location_id = str(event.get('locationID', '')) - input_hash = str(event.get('hash', 'no hash')) + input_hash = str(event.get('hash', '')) print("lat, lng: {} {}".format(lat, lng)) date_time = datetime.datetime.now() - color_code = event.get('colorCode', '1') + color_code = event.get('colorCode', 1) key = '{:.1f}_{:.1f}'.format(lat, lng) print('key: {}'.format(key)) @@ -42,16 +42,18 @@ def put_hot_spot_handler(event, context): location_id = context.aws_request_id try: + new_put_item = { + 'lat_lng': key, + 'lat': lat, + 'lng': lng, + 'date_time': "{}".format(date_time), + 'locationID': location_id, + 'colorCode': color_code + } + if input_hash: + new_put_item['hash'] = input_hash response = table.put_item( - Item={ - 'lat_lng': key, - 'lat': lat, - 'lng': lng, - 'date_time': "{}".format(date_time), - 'locationID': location_id, - 'hash': input_hash, - 'colorCode': color_code - } + Item=new_put_item ) print("response: {}".format(response)) except ClientError as e: From eff33ef4ba86d292ec811708b089ad163b06ec65 Mon Sep 17 00:00:00 2001 From: John Konderla Date: Fri, 23 Nov 2018 12:08:26 -0500 Subject: [PATCH 4/5] Removed unneeded key construction --- Source/GetHotSpots.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Source/GetHotSpots.py b/Source/GetHotSpots.py index 3344551..1945f8a 100644 --- a/Source/GetHotSpots.py +++ b/Source/GetHotSpots.py @@ -32,13 +32,11 @@ def get_hot_spots_handler(event, context): lng = Decimal('{:.1f}'.format(lng)) view_range = event['event'].get('range', 0) print('lat: {}, lng: {}'.format(lat, lng)) - lat_lng_key = Key('lat_lng').eq('{}_{}'.format(lat, lng)) date_key = Key('date_time').gt(oldest_record) print('key: {}_{}'.format(lat, lng)) response = {"Items": []} try: - # if view_range: floor = view_range outter = 0 - view_range print('floor: {}'.format(floor)) From 2b2c98032d772324c4fe74d01de011f320a0346d Mon Sep 17 00:00:00 2001 From: John Konderla Date: Sun, 25 Nov 2018 13:15:56 -0500 Subject: [PATCH 5/5] Added some additional functionality for dot replacement --- Infrastructure/PutHotSpot.template | 1 + Source/PutHotSpot.py | 92 +++++++++++++++++++----------- 2 files changed, 59 insertions(+), 34 deletions(-) diff --git a/Infrastructure/PutHotSpot.template b/Infrastructure/PutHotSpot.template index 98b57c2..c699758 100644 --- a/Infrastructure/PutHotSpot.template +++ b/Infrastructure/PutHotSpot.template @@ -65,6 +65,7 @@ Resources: Action: - dynamodb:PutItem - dynamodb:Query + - dynamodb:DeleteItem Resource: !Ref HotSpotDynamoDbTalbeArn - Effect: Allow Action: diff --git a/Source/PutHotSpot.py b/Source/PutHotSpot.py index ef5a589..64a804d 100644 --- a/Source/PutHotSpot.py +++ b/Source/PutHotSpot.py @@ -5,42 +5,50 @@ import boto3 from botocore.exceptions import ClientError +dynamodb = boto3.resource("dynamodb") +table = dynamodb.Table(os.getenv('TABLE_NAME')) + def put_hot_spot_handler(event, context): - print('event: {}'.format(event)) - dynamodb = boto3.resource("dynamodb") - - table = dynamodb.Table(os.getenv('TABLE_NAME')) - lat = Decimal(str(event.get('lat', 35.7721))) - lng = Decimal(str(event.get('lng', -78.6441))) - location_id = str(event.get('locationID', '')) - input_hash = str(event.get('hash', '')) - print("lat, lng: {} {}".format(lat, lng)) - date_time = datetime.datetime.now() - color_code = event.get('colorCode', 1) + print('starting to read.. event: {}'.format(event)) + updated_existing_item = False + if event.get('locationID', ''): + print('found locationID: {}'.format(event.get('locationID'))) + updated_existing_item = update_existing_item(event) - key = '{:.1f}_{:.1f}'.format(lat, lng) - print('key: {}'.format(key)) - response = {} - - if location_id: - print('looking for a location: {}'.format(location_id)) - two_hours_less = datetime.timedelta(hours=2) - minus_two = date_time - two_hours_less - query_response = table.query(IndexName='locationID-date_time-index', - KeyConditionExpression='locationID = :location_id and date_time > :date_time_value', - FilterExpression='lat_lng = :lat_lng_value', - ExpressionAttributeValues={ - ":location_id": location_id, - ":date_time_value": '{}'.format(minus_two), - ':lat_lng_value': key - }) - print('response.... {}'.format(query_response)) - if not query_response['Items']: - location_id = context.aws_request_id - else: - location_id = context.aws_request_id + if not updated_existing_item: + print('item not updatedable, moving on with new locationID') + event['locationID'] = context.aws_request_id + return put_new_location(event) + + +def update_existing_item(input_dict: dict): + lat, lng, location_id, input_hash, date_time, color_code, key = unpack_values(input_dict) + + print('looking for a location: {}'.format(location_id)) + two_hours_less = datetime.timedelta(hours=2) + minus_two = date_time - two_hours_less + query_response = table.query(IndexName='locationID-date_time-index', + KeyConditionExpression='locationID = :location_id and date_time > :date_time_value', + FilterExpression='lat_lng = :lat_lng_value', + ExpressionAttributeValues={ + ":location_id": location_id, + ":date_time_value": '{}'.format(minus_two), + ':lat_lng_value': key + }) + print('response.... {}'.format(query_response)) + if not query_response['Items']: + return False + print('found Items...') + old_date_time = query_response['Items'][0]['date_time'] + del_response = table.delete_item(Key={'lat_lng': key, 'date_time': old_date_time}) + print('del response: {}'.format(del_response)) + return True + +def put_new_location(input_dict: dict): + lat, lng, location_id, input_hash, date_time, color_code, key = unpack_values(input_dict) + response = '' try: new_put_item = { 'lat_lng': key, @@ -71,6 +79,22 @@ def put_hot_spot_handler(event, context): return return_dict +def unpack_values(input_dict: dict) -> tuple: + print('event: {}'.format(input_dict)) + lat = Decimal(str(input_dict.get('lat', 35.7721))) + lng = Decimal(str(input_dict.get('lng', -78.6441))) + location_id = str(input_dict.get('locationID', '')) + input_hash = str(input_dict.get('hash', '')) + print("lat, lng: {} {}".format(lat, lng)) + date_time = datetime.datetime.now() + color_code = input_dict.get('colorCode', 1) + + key = '{:.1f}_{:.1f}'.format(lat, lng) + print('key: {}'.format(key)) + + return lat, lng, location_id, input_hash, date_time, color_code, key + + if __name__ == '__main__': class Context: def __init__(self): @@ -86,5 +110,5 @@ def __init__(self): "locationID": '123123123123' } - context = Context() - print(put_hot_spot_handler(event_dict, context)) + test_context = Context() + print(put_hot_spot_handler(event_dict, test_context))