Francesco Boi Assignment

.gitignore

@@ -0,0 +1,2 @@
+**/kafka/data/
+*.pyc

README.md

@@ -1,25 +1,37 @@
+# HOW TO RUN
+## Program execution
+Run `sudo docker-compose up --build`.
+In your host open the webrowser and to the url: https://localhost.com/weather?city=<your-city>
 
+The project comprises a kafka producer that produces weather data, a kafka consumer which uses those data, a kafka broker, a web app in Flask which responds to http get requests, nd a Redis DB for readily access weather data.
 
-# NATIX Dev Challenge
+Next step:
+- use elastic search
+- add SQL DB (SQLAlchemy)
+- do some data transformations on the consumer and publish on redis to be used for the web-app
+- add MongoDB
+- Create alert Red and manage those from a Consumer
+- add a second producer on a different city
 
-Welcome to the official **NATIX Dev Challenge** repository! 🚀
-The challenges are designed to evaluate your technical skills through facing with a practical real-world problems.
+## Pytest
+Install pytest on your host or you virtualenvironment.
+To run pytests go to `dev-challenge/weather-service` and run `pytest` or `pytest tests/`. 
 
-## 🧠 Challenges
 
-```bash
-.
-├── API-breaking-change/     
-├── weather-service/        
-└── README.md       # You are here
+# PROJECT STRUCTURE
+```
+./
+├── app
+│   ├── __init__.py: initialises Redis and Flask
+│   ├── externals.py: simulates external calls
+│   ├── routes.py: app endpoints
+│   └── structures.py: file containing custom classes for type hinting
+├── tests
+│   ├── conftest.py: config file for mocking Redis, etc.
+│   └── test_app.py: unit test file
+├── docker-compose.yaml
+├── Dockerfile
+├── pytest.ini
+├── readme.md
+└── requirements.txt
 ```
-
-## 📬 Questions?
-
-If anything is unclear or missing, feel free to reach out via the channel you received the challenge on.
-
-
-Good luck, and have fun!
-— *NATIX*
-
-

app/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.11
+
+RUN pip install --upgrade pip
+WORKDIR /app
+
+#COPY app/weather.py ./
+COPY app /app
+COPY ./structures/redis_structures.py /app
+RUN pip install -r /app/requirements.txt
+
+# Set environment variables
+#comment if not a package
+ENV FLASK_APP=app 
+ENV FLASK_RUN_HOST=0.0.0.0
+
+EXPOSE 5000
+#CMD ["flask", "run", "--host=0.0.0.0"]
+#CMD python weather.py #ok
+CMD ["flask", "--app=routes", "run",  "--host=0.0.0.0", "--port=5000"]

app/__init__.py

@@ -0,0 +1,11 @@
+"""
+Initialisation module. It initialise Flask server and Redis client
+"""
+from flask import Flask
+from redis import Redis
+
+# initialise Flask
+app = Flask(__name__)
+# initialise Redis
+redis = Redis(host="redis", port=6379)
+from app import routes

app/externals.py

@@ -0,0 +1,45 @@
+"""
+Module to simulate external calls.
+"""
+from datetime import datetime, timedelta
+import random
+from app.structures import APIWeatherData
+
+def get_weather_data(city: str):
+    """Function to simulate external API returning weather ata
+
+    Args:
+        city (str): the city for which weather data should be returned
+
+    Returns:
+        _type_: weather data
+    """
+    # after 1hr the counter must be reset
+    MAX_TIME_DELTA = timedelta(hours=1)
+    date = datetime.now()
+
+    if len(get_weather_data.calls) == 0:
+        pass # date will be added at the end
+    elif (date - get_weather_data.calls[0])>MAX_TIME_DELTA:
+        # remove old calls
+        i = 0
+        for i, call in enumerate(get_weather_data.calls):
+            if date-call<MAX_TIME_DELTA:
+                break
+        get_weather_data.calls = get_weather_data.calls[i:]
+    elif len(get_weather_data.calls)>=100:
+        return {}
+    weather_type = {0:"Cloudy", 1: "Clear", 2: "Rainy", 3: "Foggy"}
+    result: APIWeatherData  = dict()
+    result["city"] = city
+    result["result"] = list()
+    base_temp = random.randrange(-20,35)
+    temp = base_temp
+    for i in range(date.hour+1):
+        temp = temp + random.randrange(-3,3)
+        result["result"].append({ "hour": i, "temperature": str(temp)+"°C", 
+                                 "condition": weather_type[random.randrange(0,len(weather_type))] })
+    get_weather_data.calls.append(date)
+    return result
+
+get_weather_data.calls = list()

app/requirements.txt

@@ -0,0 +1,5 @@
+flask
+redis
+pytest
+pydantic
+kafka-python

app/routes.py

@@ -0,0 +1,89 @@
+"""
+Module implementing endpoints and some utilities
+"""
+from datetime import datetime
+import json
+from typing import Optional
+from flask import request, jsonify
+from app import app, redis
+from app.structures import APIWeatherData
+from app.redis_structures import Date, HourlyData, RedisEntry
+from app.externals import get_weather_data
+
+def get_last_update(last_updated: Optional[Date]=None) -> bool:
+    """
+    Function compares the passed datetime to the current
+    and return the old object if year, month, day and hour are the same, the new one if not
+    or if the input is none.
+
+    Args:
+        last_updated (dict): datetime of the last updated entry in Redis
+
+    Returns:
+        bool: true if the redis entry needs to be updated, i.e., the last entry
+              does not contain the new hour(s)
+    """
+    current_date = datetime.now()
+    current_date: Date = Date(year=current_date.year, month=current_date.month,
+                      day=current_date.day, hour=current_date.hour)
+    new_update_date = last_updated
+    if last_updated is None:
+        new_update_date = current_date
+    elif current_date.year>last_updated.year or current_date.month>last_updated.month or \
+            current_date.day>last_updated.day or current_date.hour>last_updated.hour:
+        new_update_date = current_date
+    return new_update_date
+
+
+@app.route('/weather', methods=['GET']) # By default it accepts only GET methods
+def weather():
+    """
+    Decorator for "/weather" endpoing
+    Returns:
+    """
+    try:
+        city: str = request.args.get('city').lower().capitalize()
+    except:
+        return jsonify({"result": []})
+    redis_entry: RedisEntry = redis.get(city)
+    last_weather_data = {}
+    if not redis_entry:
+        # no entry means it's the first insertion for that city
+        dict_date: Date = Date.model_validate(get_last_update())
+        new_entry: RedisEntry = RedisEntry.model_validate({"last_update": dict_date, "data": list()})
+        # ASSUMPTION: a dictionary is returned directly
+        # ASSUMPTION: external api returns same data for past hours
+        #get data from external call
+        weather_data: APIWeatherData = APIWeatherData.model_validate(get_weather_data(city))
+        for hour_weather in weather_data.result:
+            hour_weather_json = hour_weather.model_dump()
+            new_entry.data.append({int( hour_weather_json.pop("hour")) : HourlyData.model_validate(hour_weather_json)})
+        redis.set(city, new_entry.model_dump_json())
+        last_weather_data = weather_data
+    else:
+        redis_data: RedisEntry = RedisEntry.model_validate(json.loads(redis.get(city)))
+        last_update: Date = Date.model_validate(redis_data.last_update)
+        new_update_date: Date = Date.model_validate(get_last_update(last_update))
+        if last_update == new_update_date:
+            # convert data from redis represention to API
+            weather_data: APIWeatherData = APIWeatherData.model_validate(
+                {
+                    "city": city,
+                    "result": [{"hour": list(element.keys())[0], **list(element.values())[0].model_dump()} 
+                        for element in redis_data.data
+                    ]
+                }
+            )
+            last_weather_data = weather_data
+        else:
+            # call external service
+            weather_data: APIWeatherData = APIWeatherData.model_validate(get_weather_data(city))
+            last_hours_inserted: int = len(redis_data.data)-1
+            for new_hourly_data in weather_data.result[last_hours_inserted+1:]:
+                hour_weather_json = new_hourly_data.model_dump()
+                hourly_data = {hour_weather_json.pop("hour") : hour_weather_json}
+                redis_data.data.append(hourly_data)
+            redis_data.last_update = new_update_date
+            redis.set(city, redis_data.model_dump_json())
+            last_weather_data = weather_data
+    return jsonify(last_weather_data.model_dump())

app/structures.py

@@ -0,0 +1,18 @@
+"""
+Moule providing custom structures
+"""
+from typing import TypedDict, List, Dict
+from pydantic import BaseModel
+from app.redis_structures import HourlyData
+
+class APIHourlyData(HourlyData):
+    """Structure representing Hourly data returned by APIs"""
+    hour: int
+
+
+class APIWeatherData(BaseModel):
+    """
+    Class representing a 
+    """
+    city: str
+    result: List[APIHourlyData]

app/tests/conftest.py

@@ -0,0 +1,19 @@
+from unittest.mock import patch
+import pytest
+
+@pytest.fixture
+def mock_redis():
+    # Define a fake Redis cache
+    redis_cache = {}
+
+    class MockRedis:
+        def get(self, key):
+            return redis_cache.get(key)
+
+        def set(self, key, value):
+            redis_cache[key] = value
+
+    mock_instance = MockRedis()
+    # PATCH THE REDIS USED IN app.routes (not app.__init__)
+    with patch("app.routes.redis", mock_instance):
+        yield mock_instance

app/tests/test_app.py

@@ -0,0 +1,57 @@
+import pytest
+from unittest.mock import patch
+from app import app
+from app.externals import get_weather_data
+
+@pytest.fixture
+def client():
+    """Function to run the app
+
+    Yields:
+        _type_: web server instance
+    """
+    with app.test_client() as client:
+        yield client
+
+def test_weather(client, mock_redis):
+    get_weather_data.calls = list()
+    for _ in range(150):
+        response = client.get("/weather?city=Astana")
+        assert response.status_code == 200
+    assert len(get_weather_data.calls) == 1
+
+def test_weather_case_insensitive_city(client, mock_redis):
+    get_weather_data.calls = list()
+    response1 = client.get("/weather?city=Astana")
+    assert response1.status_code == 200
+    assert len(get_weather_data.calls) == 1
+    response2 = client.get("/weather?city=astana")
+    assert response2.status_code == 200
+    assert len(get_weather_data.calls) == 1
+
+def test_post_method(client):
+    """Test that the http post request is not allowed 
+
+    Args:
+        client (_type_): web server
+    """
+    response = client.post("/weather?city=Astana")
+    assert response.status_code==405
+
+def test_delete_method(client):
+    """Test that the http delete request is not allowed 
+
+    Args:
+        client (_type_): web server
+    """
+    response = client.delete("/weather?city=Astana")
+    assert response.status_code==405
+
+def test_put_method(client):
+    """Test that the http put request is not allowed 
+
+    Args:
+        client (_type_): web server
+    """
+    response = client.put("/weather?city=Astana")
+    assert response.status_code==405