diff --git a/.github/workflows/ci_dev.yaml b/.github/workflows/ci_dev.yaml index ac33d650..79ce0c85 100644 --- a/.github/workflows/ci_dev.yaml +++ b/.github/workflows/ci_dev.yaml @@ -86,7 +86,7 @@ jobs: MONGO_DB_ROOT_PASSWORD: ${{ secrets.MONGO_DB_ROOT_PASSWORD }} AWX_PASSWORD: ${{ secrets.AWX_PASSWORD }} NLP_SECRET: ${{ secrets.NLP_SECRET }} - AWS_PASSWORD: ${{ secrets.AWS_PASSWORD }} + AWS_PASSWORD: ${{ secrets.AWS_SECRET_ACCESS_KEY_COLAB }} BOT_ID_DEV: ${{ secrets.BOT_ID_DEV }} BOT_NAME_DEV: ${{ secrets.BOT_NAME_DEV }} AUTHORIZED_ROOMS_DEV: ${{ secrets.AUTHORIZED_ROOMS_DEV }} @@ -100,11 +100,11 @@ jobs: NLP_SERVER_DEV: ${{ secrets.NLP_SERVER_DEV }} VCENTER_SERVER: ${{ secrets.VCENTER_SERVER }} ADMINISTRATORS_DEV: ${{ secrets.ADMINISTRATORS_DEV }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_REGION_COLAB: ${{ secrets.AWS_REGION_COLAB }} - AWS_ACCESS_KEY_ID_COLAB: ${{ secrets.AWS_ACCESS_KEY_ID_COLAB }} - AWS_PASSWORD_COLAB: ${{ secrets.AWS_SECRET_ACCESS_KEY_COLAB }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_COLAB }} COLABOT_SECRET: ${{ secrets.COLABOT_SECRET }} + DYNAMO_TABLE: ${{ secrets.DYNAMO_TABLE_DEV }} + NETBOX_URL: ${{ secrets.NETBOX_URL_DEV }} + NETBOX_TOKEN: ${{ secrets.NETBOX_TOKEN_DEV }} run: python3 process-j2.py - name: Apply and rollout diff --git a/.github/workflows/ci_prod.yaml b/.github/workflows/ci_prod.yaml index f5cd6c75..00095f56 100644 --- a/.github/workflows/ci_prod.yaml +++ b/.github/workflows/ci_prod.yaml @@ -101,10 +101,10 @@ jobs: VCENTER_SERVER: ${{ secrets.VCENTER_SERVER }} ADMINISTRATORS_PROD: ${{ secrets.ADMINISTRATORS_PROD }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_REGION_COLAB: ${{ secrets.AWS_REGION_COLAB }} - AWS_ACCESS_KEY_ID_COLAB: ${{ secrets.AWS_ACCESS_KEY_ID_COLAB }} - AWS_PASSWORD_COLAB: ${{ secrets.AWS_SECRET_ACCESS_KEY_COLAB }} COLABOT_SECRET: ${{ secrets.COLABOT_SECRET }} + DYNAMO_TABLE: ${{ secrets.DYNAMO_TABLE_PROD }} + NETBOX_URL: ${{ secrets.NETBOX_URL_PROD }} + NETBOX_TOKEN: ${{ secrets.NETBOX_TOKEN_PROD }} run: python3 process-j2.py - name: Apply and rollout diff --git a/bot.py b/bot.py index d23a104a..982317cb 100644 --- a/bot.py +++ b/bot.py @@ -41,6 +41,8 @@ "**CML show IP addresses** > show IP addresses\n", "**CML show server utilization** > show current CPU and Memory usage\n", "**CML stop lab** > stop labs of your choice\n", + "**Request IP** > allocates a static ip address for CML\n", + "**List my IPs** > lists static IPs allocated to you for CML\n", "**Create AWS account** > create AWS COLAB account\n", "**Create VPN account** > create an AnyConnect to COLAB VPN account\n", "**Create AWS key** > create aws access key\n", @@ -306,6 +308,12 @@ async def process(self, req: Request): elif self.activity.get("text") == "delete accounts": await awx.delete_accounts(self.activity) + elif self.activity.get("text") == "request ip": + await awx.request_ip(self.activity) + + elif self.activity.get("text") == "list my ips": + await awx.list_my_ips(self.activity) + elif ( self.activity.get("text")[:3] == "cml" ): # Add searches for cml dialogue here diff --git a/colabot-manifest-dev.yaml.j2 b/colabot-manifest-dev.yaml.j2 index dd8487f9..62480514 100644 --- a/colabot-manifest-dev.yaml.j2 +++ b/colabot-manifest-dev.yaml.j2 @@ -110,7 +110,7 @@ spec: - name: ADMINISTRATORS value: {{ ADMINISTRATORS_DEV }} - name: AWS_ACCESS_KEY_ID - value: {{ AWS_ACCESS_KEY_ID_COLAB }} + value: {{ AWS_ACCESS_KEY_ID }} - name: AWS_REGION value: 'us-east-1' - name: AWS_SECRET_ACCESS_KEY @@ -121,11 +121,11 @@ spec: - name: COLABOT_SECRET value: {{ COLABOT_SECRET }} - name: AWS_DYNAMO_TABLE - value: {{ DYNAMO_TABLE_DEV }} + value: {{ DYNAMO_TABLE }} - name: NETBOX_URL - value: {{ NETBOX_URL_DEV }} + value: {{ NETBOX_URL }} - name: NETBOX_TOKEN - value: {{ NETBOX_TOKEN_DEV }} + value: {{ NETBOX_TOKEN }} --- diff --git a/colabot-manifest-prod.yaml.j2 b/colabot-manifest-prod.yaml.j2 index d3a5d5f4..79b49cba 100644 --- a/colabot-manifest-prod.yaml.j2 +++ b/colabot-manifest-prod.yaml.j2 @@ -114,11 +114,11 @@ spec: - name: COLABOT_SECRET value: {{ COLABOT_SECRET }} - name: AWS_DYNAMO_TABLE - value: {{ DYNAMO_TABLE_PROD }} + value: {{ DYNAMO_TABLE }} - name: NETBOX_URL - value: {{ NETBOX_URL_PROD }} + value: {{ NETBOX_URL }} - name: NETBOX_TOKEN - value: {{ NETBOX_TOKEN_PROD }} + value: {{ NETBOX_TOKEN }} --- diff --git a/colabot-secrets-dev.yaml.j2 b/colabot-secrets-dev.yaml.j2 index d4cd6226..3e5f31de 100644 --- a/colabot-secrets-dev.yaml.j2 +++ b/colabot-secrets-dev.yaml.j2 @@ -13,4 +13,4 @@ stringData: mongo_initb_root_password: {{ MONGO_DB_ROOT_PASSWORD }} awx_password: {{ AWX_PASSWORD }} nlp_secret: {{ NLP_SECRET }} - aws_password: {{ AWS_PASSWORD_COLAB }} + aws_password: {{ AWS_PASSWORD }} diff --git a/config.py b/config.py index 27ceda12..d99ca103 100644 --- a/config.py +++ b/config.py @@ -48,4 +48,3 @@ class DefaultConfig: COLABOT_CYPHER = os.environ.get("COLABOT_SECRET") NETBOX_URL = os.environ.get("NETBOX_URL") NETBOX_TOKEN = os.environ.get("NETBOX_TOKEN") - diff --git a/features/awx.py b/features/awx.py index 8cb8ddf8..318e095b 100644 --- a/features/awx.py +++ b/features/awx.py @@ -5,12 +5,14 @@ import re import tempfile from datetime import datetime, date +import ipaddress from cryptography.fernet import Fernet import aiohttp import pymongo import urllib3 import boto3 -from boto3.dynamodb.conditions import Key +from boto3.dynamodb.conditions import Key, Attr +import pynetbox import yaml from virl2_client import ClientLibrary from jinja2 import Template @@ -640,16 +642,7 @@ async def handle_labbing_card(activity): labs_to_save = [] labs_to_delete = [] - dynamodb = boto3.resource( - "dynamodb", - region_name=CONFIG.AWS_REGION, # TODO change these from colab when going to prod - aws_access_key_id=CONFIG.AWS_ACCESS_KEY_ID, - aws_secret_access_key=CONFIG.AWS_SECRET_ACCESS_KEY, - ) - - table = dynamodb.Table( - "colab_directory" # Table Name - ) # TODO remove dev extension when pushing to prod + table = get_dynamo_colab_table() cml_server = CONFIG.SERVER_LIST.split(",")[0] user_and_domain = user_email.split("@") @@ -1083,3 +1076,234 @@ async def get_iam_user(iam_username, iam=None): return return user + + +async def request_ip(activity): + """Allocates a static ip from netbox for cml lab use""" + + logging.info("Start Request IP") + ip_limit = 10 + nb_url = str(CONFIG.NETBOX_URL) + nb_token = str(CONFIG.NETBOX_TOKEN) + username = activity["sender_email"].split("@")[0] + + ## APIs + nb = pynetbox.api(nb_url, nb_token) + webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) + table = get_dynamo_colab_table() + + ## make sure user under ip limit + ip_addresses = get_ips_dynamo(table, activity["sender_email"]) + + if len(ip_addresses) >= ip_limit: + logging.info("User %s has reached limit of %s", username, str(ip_limit)) + message = dict( + text=f"You have reached the limit of { ip_limit } reserved ip addresses", + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + return False + + # Find static ip pool on netbox + ip_ranges = nb.ipam.ip_ranges.all() + ip_range = None + for ip_range in ip_ranges: + if "static ips" in ip_range["description"].lower(): + break + ip_range = None + + if ip_range is None: + message = dict( + text="No IP pool could be found on Netbox", + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + return False + logging.info("Found IP range %s", str(ip_range)) + + # ip_type = ip_range.family.label + # check if ipv4 or 6 - below assumes 4 + + address = get_available_ip(nb, ip_range) + logging.info("Got IP address %s", str(address)) + + if address is None: + message = dict( + text="There are no more ips available", + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + return False + + # assign to user on netbox + address.description = username + address.status = "reserved" + address.save() + logging.info("Saved IP on netbox") + + ## insert ip into database + update_ip_dynamo(table, activity["sender_email"], address) + + # message user + message = dict( + text=f"New static IP Address assigned: { str(address) }", + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + + return True + + +async def list_my_ips(activity): + """Lists static ip addresses allocated to a user""" + + logging.info("START listing ips") + + ## APIs + webex = WebExClient(webex_bot_token=activity["webex_bot_token"]) + table = get_dynamo_colab_table() + + ## Retrieve IPs from database + ip_addresses = get_ips_dynamo(table, activity["sender_email"]) + + # check if ip_address field not there + if not bool(ip_addresses): + logging.info( + "User %s does not have any ips", activity["sender_email"].split("@")[0] + ) + + message = dict( + text="You do not currently have any allocated IPs", + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + return False + + # get all IPs + markdown = "" + for ip_address, ip_data in ip_addresses.items(): + last_seen = ( + datetime.today() - datetime.fromtimestamp(int(ip_data["date_last_used"])) + ).days + markdown += f"{ ip_address }: Last seen { last_seen } days ago\n" + + # send message + message = dict( + text=markdown, + toPersonId=activity["sender"], + ) + await webex.post_message_to_webex(message) + return True + + +def get_ipv4_creation_dict(ip_address: str): + """Helper function for static ip requests""" + return {"family": 4, "address": ip_address, "vrf": None} + + +def get_available_ip( + nb: pynetbox.core.api.Api, ip_range: pynetbox.models.ipam.IpRanges +): + """Returns the next available ip address as a Netbox ip object""" + + logging.info("Finding next available ip") + start_address = ip_range.start_address + mask = start_address[-3:] + net = ipaddress.ip_network(start_address, False) + + # Find first available ip + valid_ip = False + for ip in net: + ip = str(ip) + mask + + # make sure to only check at start of range, not start of net + if ip == start_address: + valid_ip = True + if not valid_ip: + continue + + address = nb.ipam.ip_addresses.get(address=ip) + if address is None: + # IP not made on netbox - so create + address = nb.ipam.ip_addresses.create(get_ipv4_creation_dict(ip)) + break + + # ip created but not assigned - what we want + if address.description == "" and address.status == "active": + break + + address = None + + return address + + +def update_ip_dynamo( + table, + user_email: str, + ip_address: str, + date_string: str = None, +): + """Creates or updates a ip address with the date string in dynamo""" + + if date_string is None: + date_string = str(int(datetime.timestamp(datetime.now()))) + + # check to see if ip field already there + response = table.query( + KeyConditionExpression=Key("email").eq(user_email), + FilterExpression=Attr("ip_addresses").exists(), + ) + + # create ip field map if it doesn't exist + if response["Count"] == 0: + table.update_item( + Key={"email": user_email}, + UpdateExpression="SET #ip_addresses= :value", + ExpressionAttributeNames={"#ip_addresses": "ip_addresses"}, + ExpressionAttributeValues={":value": {}}, + ) + + # insert new ip + table.update_item( + Key={"email": user_email}, + UpdateExpression="SET #ip_addresses.#ip_address= :ip_data", + ExpressionAttributeNames={ + "#ip_addresses": "ip_addresses", + "#ip_address": str(ip_address), + }, + ExpressionAttributeValues={ + ":ip_data": { + "date_last_used": date_string, + } + }, + ) + + logging.info("Updated IP on dynamo") + + +def get_ips_dynamo(table, user_email: str): + """Returns the ip addresses associated with a user""" + + logging.info("Retrieving IPs") + + response = table.query(KeyConditionExpression=Key("email").eq(user_email)) + + if "ip_addresses" not in response["Items"][0]: + return {} + + return response["Items"][0]["ip_addresses"] + + +def get_dynamo_colab_table(): + """Returns dynamo colab table""" + + dynamodb = boto3.resource( + "dynamodb", + region_name=CONFIG.AWS_REGION, # TODO change these from colab when going to prod + aws_access_key_id=CONFIG.AWS_ACCESS_KEY_ID, + aws_secret_access_key=CONFIG.AWS_SECRET_ACCESS_KEY, + ) + + table = dynamodb.Table(CONFIG.AWS_DYNAMO_TABLE) # Table Name + + return table diff --git a/requirements.txt b/requirements.txt index 9460abd1..5a92ac4b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,5 @@ webexteamssdk==1.6 virl2_client==2.4.0 PyYAML==5.4 cryptography==39.0.1 +ipaddress==1.0.23 +pynetbox==7.0.1