diff --git a/.flake8 b/.flake8 index 05a09f3..a960d69 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,4 @@ [flake8] -max-line-length = 88 +max-line-length = 100 extend-ignore = E203, W503 exclude = .venv,venv,migrations,__pycache__,.git diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f4586ac --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 [Anyer Moreno Alcaraz] + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 87d183b..ce07ad6 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,106 @@ # MetaAppCollector - Backend -MetaAppCollector és una eina per agregar metadades d'aplicacions mòbils des de diverses fonts externes, com botigues d'apps, xarxes socials i motors de cerca. +**MetaAppCollector** és una eina *open source* per monitoritzar aplicacions mòbils agregant informació pública de diverses fonts com botigues d'apps i xarxes socials. -Aquest projecte és el backend del sistema, desenvolupat amb **Django**. +> 🧪 Desenvolupada amb Django, Celery, Redis i Docker. Pensada per a desenvolupadors, investigadors i equips de màrqueting que volen entendre millor la visibilitat de les seves apps. -## 🔧 Requisits +![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg) -- Python 3.12 -- Django 5.1.7 -- Git +--- -## 🚀 Instal·lació +## 🔧 Funcionalitats principals + +- 🔍 Recuperació de metadades i reviews d'aplicacions mòbils (Google Play, etc.) +- 📊 Agregació de mètriques d'interacció i visibilitat +- ⏱️ Polling periòdic configurable per recollida automàtica +- 🧠 Arquitectura extensible amb adaptadors dinàmics +- 🐇 Execució asíncrona de tasques amb Celery + Redis + +--- + +## 🚀 Posada en marxa amb Docker + +Assegura’t de tenir instal·lats: + +- [Docker](https://www.docker.com/) +- [Docker Compose](https://docs.docker.com/compose/) + +### 1. Clona el projecte + +```bash +git clone https://github.com/el-teu-usuari/metaappcollector-backend.git +cd metaappcollector-backend +``` + +### 2. Configura variables d'entorn + +Crea un fitxer .env a l'arrel amb el següent contingut mínim: + +```bash +DJANGO_SECRET_KEY=your-secret-key +DJANGO_DEBUG=True +``` + +### 3. Executa els serveis ```bash -git clone git@github.com:Anyerrr/MetaAppCollector-Backend.git -cd MetaAppCollector-Backend -python -m venv .venv -.venv\Scripts\activate -pip install -r requirements.txt +docker compose up --build ``` +Això iniciarà: -## 🛠 ️ Execució +- El servidor Django a http://localhost:8000 +- El worker de Celery +- El Beat de Celery (scheduler) +- Redis com a broker + +## 🧪 Prova Celery +Obre una shell dins del contenidor web: +```bash +docker compose exec web python manage.py shell +``` +I prova una tasca: +```bash +from polling.tasks import prueba_celery +prueba_celery.delay() +``` +Pots veure els resultats al log del worker: +```bash +docker compose logs -f celery +``` + +## 🧱 Estructura del projecte ```bash -python manage.py runserver +├── polling/ # Lògica de polling i tasques Celery +├── metric/ # Agregació de mètriques +├── review/ # Gestió de reviews +├── source/ # Fonts de dades integrades +├── app/ # Aplicacions mòbils registrades +├── config/ # Configuració general de Django i Celery +├── docker-compose.yml # Definició dels serveis ``` -## 🧪 Proves +## 📚 Comandes útils ```bash -python manage.py test -``` \ No newline at end of file +docker compose exec web python manage.py migrate # Aplicar migracions +docker compose exec web python manage.py createsuperuser +docker compose exec web python manage.py shell # Obrir shell +``` + +## 🤝 Contribució +Les contribucions són benvingudes! Si vols col·laborar: + +1. Fes un fork del repositori +2. Crea una branca nova (git checkout -b feature/nou-adaptador) +3. Fes els teus canvis i commiteja (git commit -am 'Afegit adaptador per X') +4. Puja la branca (git push origin feature/nou-adaptador) +5. Fes una pull request 🙌 + +Abans d’enviar, assegura’t de seguir l’estil de codi del projecte i escriure tests quan sigui necessari. + + +## ✉️ Contacte +Per dubtes tècnics o col·laboració, contacta amb el desenvolupador principal: +📧 a.moreno@estudiantat.upc.edu + +## 📄 Llicència +Aquest projecte està llicenciat sota la MIT License – consulta el fitxer LICENSE per a més informació. \ No newline at end of file diff --git a/app/migrations/0002_app_code.py b/app/migrations/0002_app_code.py new file mode 100644 index 0000000..f9cc194 --- /dev/null +++ b/app/migrations/0002_app_code.py @@ -0,0 +1,19 @@ +# Generated by Django 5.1.7 on 2025-04-05 14:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("app", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="app", + name="code", + field=models.CharField(default="default_code", max_length=100, unique=True), + preserve_default=False, + ), + ] diff --git a/app/migrations/0003_alter_app_developer.py b/app/migrations/0003_alter_app_developer.py new file mode 100644 index 0000000..099d203 --- /dev/null +++ b/app/migrations/0003_alter_app_developer.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.7 on 2025-04-27 12:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("app", "0002_app_code"), + ] + + operations = [ + migrations.AlterField( + model_name="app", + name="developer", + field=models.CharField(max_length=100, null=True), + ), + ] diff --git a/app/models.py b/app/models.py index 4b8bd84..bf7c3bf 100644 --- a/app/models.py +++ b/app/models.py @@ -2,11 +2,12 @@ class App(models.Model): + code = models.CharField(max_length=100, unique=True) name = models.CharField(max_length=200) description = models.TextField() appstore_id = models.CharField(max_length=100, null=True, unique=True) playstore_id = models.CharField(max_length=100, null=True, unique=True) - developer = models.CharField(max_length=100, null=True, unique=True) + developer = models.CharField(max_length=100, null=True) available_on_ios = models.BooleanField(default=False) available_on_android = models.BooleanField(default=False) pegi_rating = models.CharField(max_length=10, null=True) # PEGI rating diff --git a/app/repositories.py b/app/repositories.py new file mode 100644 index 0000000..0a5d2db --- /dev/null +++ b/app/repositories.py @@ -0,0 +1,26 @@ +from .models import App + + +class AppRepository: + @staticmethod + def get_all(): + return App.objects.all() + + @staticmethod + def get_by_id(app_id): + return App.objects.get(id=app_id) + + @staticmethod + def create(data): + return App.objects.create(**data) + + @staticmethod + def update(instance, data): + for attr, value in data.items(): + setattr(instance, attr, value) + instance.save() + return instance + + @staticmethod + def delete(instance): + instance.delete() diff --git a/app/serializers.py b/app/serializers.py new file mode 100644 index 0000000..c8834aa --- /dev/null +++ b/app/serializers.py @@ -0,0 +1,28 @@ +from rest_framework import serializers + +from .models import App + + +class AppSerializer(serializers.ModelSerializer): + class Meta: + model = App + fields = [ + "id", + "code", + "name", + "description", + "appstore_id", + "playstore_id", + "developer", + "available_on_ios", + "available_on_android", + "pegi_rating", + "release_date", + "min_ios_version", + ] + + +class AppCreateSerializer(serializers.ModelSerializer): + class Meta: + model = App + fields = ["code", "name", "description", "appstore_id", "playstore_id"] diff --git a/app/services.py b/app/services.py new file mode 100644 index 0000000..11b684a --- /dev/null +++ b/app/services.py @@ -0,0 +1,93 @@ +from datetime import datetime + +from django.core.exceptions import ObjectDoesNotExist +from rest_framework.exceptions import NotFound + +from .repositories import AppRepository + + +class AppService: + def __init__(self, itunes_adapter=None, google_play_adapter=None): + self.repo = AppRepository() + self._itunes_adapter = itunes_adapter + self._google_play_adapter = google_play_adapter + + @property + def itunes_adapter(self): + if self._itunes_adapter is None: + from source.adapters.itunes import ItunesSearchAPIAdapter + + self._itunes_adapter = ItunesSearchAPIAdapter() + return self._itunes_adapter + + @property + def google_play_adapter(self): + if self._google_play_adapter is None: + from source.adapters.google_play_scraper import GooglePlayScraperAdapter + + self._google_play_adapter = GooglePlayScraperAdapter() + return self._google_play_adapter + + def list_apps(self): + return self.repo.get_all() + + def get_app(self, app_id): + try: + return self.repo.get_by_id(app_id) + except ObjectDoesNotExist: + raise NotFound(f"The app with ID '{app_id}' is not registered.") + + def create_app(self, validated_data): + self._fetch_appstore_data(validated_data) + self._fetch_playstore_data(validated_data) + return self.repo.create(validated_data) + + def _fetch_appstore_data(self, validated_data): + if not validated_data.get("appstore_id"): + return + + result = self.itunes_adapter.lookup_app(validated_data["appstore_id"]) + if not result: + return + + validated_data["available_on_ios"] = True + validated_data["developer"] = result.get("artistName") + validated_data["pegi_rating"] = result.get("contentAdvisoryRating") + validated_data["min_ios_version"] = result.get("minimumOsVersion") + + released_str = result.get("releaseDate") + if released_str: + try: + if "T" in released_str: + release_date = datetime.fromisoformat(released_str.replace("Z", "")).date() + validated_data["release_date"] = release_date + except Exception as e: + print(f"Error parsejant release_date (AppStore): {released_str} ({e})") + + def _fetch_playstore_data(self, validated_data): + if not validated_data.get("playstore_id"): + return + + result = self.google_play_adapter.lookup_app(validated_data["playstore_id"]) + if not result: + return + + validated_data["available_on_android"] = True + validated_data["developer"] = validated_data.get("developer") or result.get("developer") + validated_data["pegi_rating"] = validated_data.get("pegi_rating") or result.get( + "contentRating" + ) + + released_str = result.get("released") + if released_str and not validated_data.get("release_date"): + try: + release_date = datetime.strptime(released_str, "%b %d, %Y").date() + validated_data["release_date"] = release_date + except ValueError: + print(f"Format de data invàlid (PlayStore): {released_str}") + + def update_app(self, instance, validated_data): + return self.repo.update(instance, validated_data) + + def delete_app(self, instance): + return self.repo.delete(instance) diff --git a/app/tests.py b/app/tests.py index e69de29..f359d1c 100644 --- a/app/tests.py +++ b/app/tests.py @@ -0,0 +1,18 @@ +import pytest + +from app.models import App + + +@pytest.mark.django_db +def test_create_source(): + a = App.objects.create( + code="test_app", + name="Test App", + description="Descripction of the test app", + appstore_id="123456", + ) + assert a.pk is not None + assert a.code == "test_app" + assert a.name == "Test App" + assert a.description == "Descripction of the test app" + assert a.appstore_id == "123456" diff --git a/app/urls.py b/app/urls.py new file mode 100644 index 0000000..8118486 --- /dev/null +++ b/app/urls.py @@ -0,0 +1,11 @@ +from django.urls import include, path +from rest_framework.routers import DefaultRouter + +from .views import AppViewSet + +router = DefaultRouter() +router.register(r"apps", AppViewSet, basename="app") + +urlpatterns = [ + path("", include(router.urls)), +] diff --git a/app/views.py b/app/views.py index e69de29..6ce18c7 100644 --- a/app/views.py +++ b/app/views.py @@ -0,0 +1,71 @@ +from drf_spectacular.utils import OpenApiParameter, extend_schema +from rest_framework import status, viewsets +from rest_framework.response import Response + +from .serializers import AppCreateSerializer, AppSerializer +from .services import AppService + + +class AppViewSet(viewsets.ViewSet): + service = AppService() + + @extend_schema( + responses=AppSerializer(many=True), + tags=["Apps"], + ) + def list(self, request): + apps = self.service.list_apps() + serializer = AppSerializer(apps, many=True) + return Response(serializer.data) + + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses=AppSerializer, + tags=["Apps"], + ) + def retrieve(self, request, pk=None): + app = self.service.get_app(pk) + serializer = AppSerializer(app) + return Response(serializer.data) + + @extend_schema( + request=AppCreateSerializer, + responses=AppSerializer, + description=( + "Creates a new app.\n\n" + "The fields 'available_on_ios', 'available_on_android', 'pegi_rating', " + "'release_date', and 'min_ios_version' will be automatically populated " + "by fetching data from the App Store and/or Google Play APIs based " + "on the provided identifiers.\n\n" + "You do not need to manually provide these fields when creating an app." + ), + tags=["Apps"], + ) + def create(self, request): + print("REQUEST DATA:", request.data) + serializer = AppCreateSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + app = self.service.create_app(serializer.validated_data) + return Response(AppSerializer(app).data, status=status.HTTP_201_CREATED) + + @extend_schema( + request=AppSerializer, + responses=AppSerializer, + tags=["Apps"], + ) + def update(self, request, pk=None): + app = self.service.get_app(pk) + serializer = AppSerializer(app, data=request.data) + serializer.is_valid(raise_exception=True) + updated_app = self.service.update_app(app, serializer.validated_data) + return Response(AppSerializer(updated_app).data) + + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses={204: None}, + tags=["Apps"], + ) + def destroy(self, request, pk=None): + app = self.service.get_app(pk) + self.service.delete_app(app) + return Response(status=status.HTTP_204_NO_CONTENT) diff --git a/config/__init__.py b/config/__init__.py index e69de29..53f4ccb 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ("celery_app",) diff --git a/config/__pycache__/__init__.cpython-312.pyc b/config/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 52dcab1..0000000 Binary files a/config/__pycache__/__init__.cpython-312.pyc and /dev/null differ diff --git a/config/__pycache__/settings.cpython-312.pyc b/config/__pycache__/settings.cpython-312.pyc deleted file mode 100644 index c6b8494..0000000 Binary files a/config/__pycache__/settings.cpython-312.pyc and /dev/null differ diff --git a/config/__pycache__/urls.cpython-312.pyc b/config/__pycache__/urls.cpython-312.pyc deleted file mode 100644 index 9c0c174..0000000 Binary files a/config/__pycache__/urls.cpython-312.pyc and /dev/null differ diff --git a/config/__pycache__/wsgi.cpython-312.pyc b/config/__pycache__/wsgi.cpython-312.pyc deleted file mode 100644 index 3660406..0000000 Binary files a/config/__pycache__/wsgi.cpython-312.pyc and /dev/null differ diff --git a/config/celery.py b/config/celery.py new file mode 100644 index 0000000..60888ac --- /dev/null +++ b/config/celery.py @@ -0,0 +1,9 @@ +import os + +from celery import Celery + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings") + +app = Celery("metaappcollector") +app.config_from_object("django.conf:settings", namespace="CELERY") +app.autodiscover_tasks() diff --git a/config/settings.py b/config/settings.py index 509a3a0..2a83d36 100644 --- a/config/settings.py +++ b/config/settings.py @@ -49,18 +49,29 @@ "source", "review", "metric", + "polling", + "drf_spectacular", + "django_celery_beat", + "corsheaders", ] MIDDLEWARE = [ + "corsheaders.middleware.CorsMiddleware", + "django.middleware.common.CommonMiddleware", "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", - "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", ] +CORS_ALLOWED_ORIGINS = [ + "https://editor.swagger.io", + "http://localhost:3000", + "https://hn13e-puma-hackers.github.io", +] + ROOT_URLCONF = "config.urls" TEMPLATES = [ @@ -98,8 +109,7 @@ AUTH_PASSWORD_VALIDATORS = [ { - "NAME": "django.contrib.auth.password_validation." - "UserAttributeSimilarityValidator", + "NAME": "django.contrib.auth.password_validation." "UserAttributeSimilarityValidator", }, { "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", @@ -134,3 +144,10 @@ # https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + +REST_FRAMEWORK = { + "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", +} + +CELERY_BROKER_URL = "redis://redis:6379/0" +CELERY_RESULT_BACKEND = "redis://redis:6379/0" diff --git a/config/urls.py b/config/urls.py index a7369d4..a47e27e 100644 --- a/config/urls.py +++ b/config/urls.py @@ -17,7 +17,8 @@ from django.contrib import admin from django.http import HttpResponse -from django.urls import path +from django.urls import include, path +from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView def home(request): @@ -27,4 +28,11 @@ def home(request): urlpatterns = [ path("admin/", admin.site.urls), path("", home), + path("api/schema/", SpectacularAPIView.as_view(), name="schema"), + path("api/docs/", SpectacularSwaggerView.as_view(url_name="schema"), name="swagger-ui"), + path("api/", include("app.urls")), + path("api/", include("source.urls")), + path("api/", include("review.urls")), + path("api/", include("metric.urls")), + path("api/polling/", include("polling.urls")), ] diff --git a/docker-compose.yml b/docker-compose.yml index 903bb4e..bbd7dc4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,57 @@ - services: + redis: + image: redis:7 + container_name: redis + ports: + - "6379:6379" + web: build: . + container_name: django + command: python manage.py runserver 0.0.0.0:8000 + volumes: + - .:/app ports: - "8000:8000" + depends_on: + - redis + env_file: + - .env + + celery: + build: . + container_name: celery_worker + command: celery -A config worker --loglevel=info volumes: - .:/app - env_file: - - .env.prod + depends_on: + - redis + + beat: + build: . + container_name: celery_beat + command: celery -A config beat --loglevel=info --scheduler django_celery_beat.schedulers:DatabaseScheduler + volumes: + - .:/app + depends_on: + - redis + + re-miner-polarity: + build: + context: ../RE-Miner-polarity-analysis + ports: + - "3010:3010" + command: uvicorn main:app --host 0.0.0.0 --port 3010 + volumes: + - ../RE-Miner-polarity-analysis:/app + working_dir: /app + + re-miner-type: + build: + context: ../RE-Miner-type-analysis + ports: + - "3011:3011" + command: uvicorn main:app --host 0.0.0.0 --port 3011 + volumes: + - ../RE-Miner-type-analysis:/app + working_dir: /app \ No newline at end of file diff --git a/metric/admin.py b/metric/admin.py index c7c8297..fceef0b 100644 --- a/metric/admin.py +++ b/metric/admin.py @@ -3,4 +3,10 @@ from .models import Metric, MetricValue admin.site.register(Metric) -admin.site.register(MetricValue) + + +@admin.register(MetricValue) +class MetricValueAdmin(admin.ModelAdmin): + list_display = ("app", "metric", "source", "value", "retrieved_at") + ordering = ("-retrieved_at", "-metric") + list_filter = ("metric", "source", "retrieved_at") diff --git a/metric/constants/__init__.py b/metric/constants/__init__.py new file mode 100644 index 0000000..232a183 --- /dev/null +++ b/metric/constants/__init__.py @@ -0,0 +1,7 @@ +class MetricCodes: + AVERAGE_RATING = "average_rating" + TOTAL_REVIEWS = "total_reviews" + DAILY_NEWS_BLOG_MENTIONS = "daily_news_blog_mentions" + DAILY_SOCIAL_NETWORK_MENTIONS = "daily_social_network_mentions" + TOTAL_DOWNLOADS = "total_downloads" + LAST_UPDATE_DATE = "last_update_date" diff --git a/metric/constants/value_types.py b/metric/constants/value_types.py new file mode 100644 index 0000000..752b940 --- /dev/null +++ b/metric/constants/value_types.py @@ -0,0 +1,9 @@ +from django.db import models + + +class MetricValueType(models.TextChoices): + STRING = "string", "String" + INTEGER = "integer", "Integer" + FLOAT = "float", "Float" + DATE = "date", "Date" + BOOLEAN = "boolean", "Boolean" diff --git a/metric/migrations/0001_initial.py b/metric/migrations/0001_initial.py index 8d1c2f4..146a52d 100644 --- a/metric/migrations/0001_initial.py +++ b/metric/migrations/0001_initial.py @@ -59,9 +59,7 @@ class Migration(migrations.Migration): ("retrieved_at", models.DateTimeField(auto_now_add=True)), ( "app", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, to="app.app" - ), + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="app.app"), ), ( "metric", diff --git a/metric/migrations/0002_metric_code.py b/metric/migrations/0002_metric_code.py new file mode 100644 index 0000000..a77834c --- /dev/null +++ b/metric/migrations/0002_metric_code.py @@ -0,0 +1,19 @@ +# Generated by Django 5.1.7 on 2025-04-04 18:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("metric", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="metric", + name="code", + field=models.CharField(default="default_code", max_length=100), + preserve_default=False, + ), + ] diff --git a/metric/migrations/0003_alter_metric_code.py b/metric/migrations/0003_alter_metric_code.py new file mode 100644 index 0000000..154fa17 --- /dev/null +++ b/metric/migrations/0003_alter_metric_code.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.7 on 2025-04-05 14:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("metric", "0002_metric_code"), + ] + + operations = [ + migrations.AlterField( + model_name="metric", + name="code", + field=models.CharField(max_length=100, unique=True), + ), + ] diff --git a/metric/migrations/0004_alter_metricvalue_source.py b/metric/migrations/0004_alter_metricvalue_source.py new file mode 100644 index 0000000..f6e8e26 --- /dev/null +++ b/metric/migrations/0004_alter_metricvalue_source.py @@ -0,0 +1,25 @@ +# Generated by Django 5.1.7 on 2025-05-04 17:49 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("metric", "0003_alter_metric_code"), + ("source", "0003_source_metrics"), + ] + + operations = [ + migrations.AlterField( + model_name="metricvalue", + name="source", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="source.source", + ), + ), + ] diff --git a/metric/migrations/0005_add_is_internal.py b/metric/migrations/0005_add_is_internal.py new file mode 100644 index 0000000..ee3b007 --- /dev/null +++ b/metric/migrations/0005_add_is_internal.py @@ -0,0 +1,16 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("metric", "0004_alter_metricvalue_source"), + ] + + operations = [ + migrations.AddField( + model_name="metric", + name="is_internal", + field=models.BooleanField(default=False), + ), + ] diff --git a/metric/models.py b/metric/models.py index 72ad355..2c6bb8d 100644 --- a/metric/models.py +++ b/metric/models.py @@ -1,21 +1,15 @@ from django.db import models from app.models import App -from source.models import Source - - -class MetricValueType(models.TextChoices): - STRING = "string", "String" - INTEGER = "integer", "Integer" - FLOAT = "float", "Float" - DATE = "date", "Date" - BOOLEAN = "boolean", "Boolean" +from metric.constants.value_types import MetricValueType class Metric(models.Model): + code = models.CharField(max_length=100, unique=True) name = models.CharField(max_length=100) description = models.TextField() value_type = models.CharField(max_length=10, choices=MetricValueType.choices) + is_internal = models.BooleanField(default=False) def __str__(self): return self.name @@ -24,9 +18,12 @@ def __str__(self): class MetricValue(models.Model): app = models.ForeignKey(App, on_delete=models.CASCADE) metric = models.ForeignKey(Metric, on_delete=models.CASCADE) - source = models.ForeignKey(Source, on_delete=models.CASCADE) + source = models.ForeignKey("source.Source", on_delete=models.CASCADE, null=True, blank=True) value = models.CharField(max_length=255) retrieved_at = models.DateTimeField(auto_now_add=True) def __str__(self): - return f"{self.metric.name} - {self.source.name} - {self.app.name}" + return ( + f"{self.metric.name} - {self.source.name if self.source else 'Internal'} " + f"- {self.app.name}" + ) diff --git a/metric/registry/source_registry.py b/metric/registry/source_registry.py deleted file mode 100644 index d119020..0000000 --- a/metric/registry/source_registry.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import List - -from source.adapters.base import SourceAdapter - - -class SourceRegistry: - def __init__(self, adapters: List[SourceAdapter]): - self.adapters = adapters - - def get_sources(self) -> List[SourceAdapter]: - return self.adapters - - def get_for_metric(self, metric: str) -> List[SourceAdapter]: - return [adapter for adapter in self.adapters if adapter.supports_metric(metric)] diff --git a/metric/repositories.py b/metric/repositories.py new file mode 100644 index 0000000..60e31c7 --- /dev/null +++ b/metric/repositories.py @@ -0,0 +1,63 @@ +from .models import Metric, MetricValue + + +class MetricRepository: + @staticmethod + def get_all(): + return Metric.objects.all() + + @staticmethod + def get_by_id(pk): + return Metric.objects.get(id=pk) + + @staticmethod + def create(data): + return Metric.objects.create(**data) + + @staticmethod + def update(instance, data): + for attr, value in data.items(): + setattr(instance, attr, value) + instance.save() + return instance + + @staticmethod + def delete(instance): + instance.delete() + + @staticmethod + def add_sources(instance, sources_ids): + instance.sources.add(*sources_ids) + instance.save() + return instance + + @staticmethod + def remove_sources(instance, sources_ids): + instance.sources.remove(*sources_ids) + instance.save() + return instance + + +class MetricValueRepository: + @staticmethod + def get_all(): + return MetricValue.objects.all() + + @staticmethod + def get_by_id(pk): + return MetricValue.objects.get(id=pk) + + @staticmethod + def create(data): + return MetricValue.objects.create(**data) + + @staticmethod + def update(instance, data): + for attr, value in data.items(): + setattr(instance, attr, value) + instance.save() + return instance + + @staticmethod + def delete(instance): + instance.delete() diff --git a/metric/serializers.py b/metric/serializers.py new file mode 100644 index 0000000..5e21823 --- /dev/null +++ b/metric/serializers.py @@ -0,0 +1,34 @@ +from rest_framework import serializers + +from .constants.value_types import MetricValueType +from .models import Metric, MetricValue + + +class MetricSerializer(serializers.ModelSerializer): + value_type = serializers.ChoiceField(choices=MetricValueType.choices) + + class Meta: + model = Metric + fields = ["id", "code", "name", "description", "value_type", "sources"] + read_only_fields = ["sources"] + + +class MetricValueSerializer(serializers.ModelSerializer): + class Meta: + model = MetricValue + fields = [ + "id", + "app", + "metric", + "source", + "value", + "retrieved_at", + ] + + read_only_fields = fields + + +class LinkMetricsSerializer(serializers.Serializer): + metrics = serializers.ListField( + child=serializers.IntegerField(), help_text="List of metric IDs to link to the source." + ) diff --git a/metric/services.py b/metric/services.py new file mode 100644 index 0000000..80616df --- /dev/null +++ b/metric/services.py @@ -0,0 +1,56 @@ +from django.core.exceptions import ObjectDoesNotExist +from rest_framework.exceptions import NotFound + +from .repositories import MetricRepository, MetricValueRepository + + +class MetricService: + def __init__(self): + self.repo = MetricRepository() + + def list_metrics(self): + return self.repo.get_all() + + def get_metric(self, pk): + try: + return self.repo.get_by_id(pk) + except ObjectDoesNotExist: + raise NotFound(f"The metric with ID '{pk}' is not registered.") + + def create_metric(self, validated_data): + return self.repo.create(validated_data) + + def update_metric(self, instance, validated_data): + return self.repo.update(instance, validated_data) + + def delete_metric(self, instance): + return self.repo.delete(instance) + + def add_sources(self, instance, sources_ids): + return self.repo.add_sources(instance, sources_ids) + + def remove_sources(self, instance, sources_ids): + return self.repo.remove_sources(instance, sources_ids) + + +class MetricValueService: + def __init__(self): + self.repo = MetricValueRepository() + + def list_metric_values(self): + return self.repo.get_all() + + def get_metric_value(self, pk): + try: + return self.repo.get_by_id(pk) + except ObjectDoesNotExist: + raise NotFound(f"The metric_value with ID '{pk}' is not registered.") + + def create_metric_value(self, validated_data): + return self.repo.create(validated_data) + + def update_metric_value(self, instance, validated_data): + return self.repo.update(instance, validated_data) + + def delete_metric_value(self, instance): + return self.repo.delete(instance) diff --git a/metric/strategies/average_rating.py b/metric/strategies/average_rating.py deleted file mode 100644 index 88a0fac..0000000 --- a/metric/strategies/average_rating.py +++ /dev/null @@ -1,22 +0,0 @@ -from datetime import datetime - -from metric.models import MetricValue -from metric.registry.source_registry import SourceRegistry -from metric.strategies.base import MetricStrategy - - -class AverageRatingMetric(MetricStrategy): - def compute_all(self, app_id: str, registry: SourceRegistry) -> list[MetricValue]: - results = [] - for adapter in registry.get_for_metric("average_rating"): - value = adapter.fetch(app_id, "average_rating") - results.append( - MetricValue( - app_id=app_id, - metric_code="average_rating", - source_name=adapter.__class__.__name__, - value=value, - retrieved_at=datetime.now(), - ) - ) - return results diff --git a/metric/strategies/base.py b/metric/strategies/base.py deleted file mode 100644 index 290d1d1..0000000 --- a/metric/strategies/base.py +++ /dev/null @@ -1,11 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List - -from metric.models import MetricValue -from metric.registry.source_registry import SourceRegistry - - -class MetricStrategy(ABC): - @abstractmethod - def compute_all(self, app_id: str, registry: SourceRegistry) -> List[MetricValue]: - pass diff --git a/metric/strategies/generic.py b/metric/strategies/generic.py new file mode 100644 index 0000000..c4b2825 --- /dev/null +++ b/metric/strategies/generic.py @@ -0,0 +1,36 @@ +from datetime import datetime + +from app.models import App +from metric.models import Metric, MetricValue +from source.adapters.base import SourceAdapter +from source.models import Source + + +class GenericMetricStrategy: + def __init__(self, metric_code: str): + self.metric_code = metric_code + + def compute_all(self, app_id: str, adapters: list[SourceAdapter]) -> list[MetricValue]: + results = [] + + app = App.objects.get(id=app_id) + metric = Metric.objects.get(code=self.metric_code) + + for adapter in adapters: + if not adapter.supports_metric(self.metric_code): + continue + + value = adapter.fetch(app_id, self.metric_code) + + source = Source.objects.get(code=adapter.code) + + results.append( + MetricValue( + app=app, + metric=metric, + source=source, + value=value, + retrieved_at=datetime.now(), + ) + ) + return results diff --git a/metric/tests.py b/metric/tests.py index e69de29..7fe1653 100644 --- a/metric/tests.py +++ b/metric/tests.py @@ -0,0 +1,47 @@ +import pytest + +from app.models import App +from metric.constants.value_types import MetricValueType +from metric.models import Metric, MetricValue +from source.constants.source_type import SourceType +from source.models import Source + + +@pytest.mark.django_db +def test_create_metric(): + m = Metric.objects.create( + code="test_metric", + name="Test Metric", + description="Descripció de test", + value_type=MetricValueType.FLOAT, + ) + assert m.pk is not None + assert m.code == "test_metric" + + +@pytest.mark.django_db +def test_create_metric_value(): + a = App.objects.create( + code="test_app", + name="Test App", + description="Descripction of the test app", + appstore_id="123456", + ) + s = Source.objects.create( + code="test_source", + name="Test Source", + type=SourceType.API, + url="https://example.com", + ) + m = Metric.objects.create( + code="test_metric", + name="Test Metric", + description="Descripció de test", + value_type=MetricValueType.FLOAT, + ) + mv = MetricValue.objects.create(app=a, metric=m, source=s, value="123.45") + assert mv.pk is not None + assert mv.app == a + assert mv.metric == m + assert mv.source == s + assert mv.value == "123.45" diff --git a/metric/urls.py b/metric/urls.py new file mode 100644 index 0000000..9891c6c --- /dev/null +++ b/metric/urls.py @@ -0,0 +1,12 @@ +from django.urls import include, path +from rest_framework.routers import DefaultRouter + +from metric.views import MetricValueViewSet, MetricViewSet + +router = DefaultRouter() +router.register(r"metrics", MetricViewSet, basename="metric") +router.register(r"metric-values", MetricValueViewSet, basename="metric_value") + +urlpatterns = [ + path("", include(router.urls)), +] diff --git a/metric/views.py b/metric/views.py index e69de29..865a38d 100644 --- a/metric/views.py +++ b/metric/views.py @@ -0,0 +1,134 @@ +from drf_spectacular.utils import OpenApiParameter, extend_schema +from rest_framework import status, viewsets +from rest_framework.decorators import action +from rest_framework.response import Response + +from source.serializers import LinkSourceSerializer + +from .serializers import MetricSerializer, MetricValueSerializer +from .services import MetricService, MetricValueService + + +class MetricViewSet(viewsets.ViewSet): + service = MetricService() + + @extend_schema( + responses=MetricSerializer(many=True), + tags=["Metrics"], + ) + def list(self, request): + metrics = self.service.list_metrics() + serializer = MetricSerializer(metrics, many=True) + return Response(serializer.data) + + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses=MetricSerializer, + tags=["Metrics"], + ) + def retrieve(self, request, pk=None): + metric = self.service.get_metric(pk) + serializer = MetricSerializer(metric) + return Response(serializer.data) + + @extend_schema( + request=MetricSerializer, + responses=MetricSerializer, + description=( + "Creates a new data metric.\n\n" + "The 'source' field is read-only and cannot be set directly on creation." + ), + tags=["Metrics"], + ) + def create(self, request): + serializer = MetricSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + metric = self.service.create_metric(serializer.validated_data) + return Response(MetricSerializer(metric).data, status=status.HTTP_201_CREATED) + + @extend_schema( + request=MetricSerializer, + responses=MetricSerializer, + tags=["Metrics"], + ) + def update(self, request, pk=None): + metric = self.service.get_metric(pk) + serializer = MetricSerializer(metric, data=request.data) + serializer.is_valid(raise_exception=True) + updated_metric = self.service.update_metric(metric, serializer.validated_data) + return Response(MetricSerializer(updated_metric).data) + + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses={204: None}, + tags=["Metrics"], + ) + def destroy(self, request, pk=None): + metric = self.service.get_metric(pk) + self.service.delete_metric(metric) + return Response(status=status.HTTP_204_NO_CONTENT) + + @extend_schema( + request=LinkSourceSerializer, + responses=MetricSerializer, + description="Adds a list of source IDs to an existing " + "metric without removing existing ones.", + tags=["Metrics"], + ) + @action(detail=True, methods=["post"], url_path="sources") + def add_metrics(self, request, pk=None): + sources_ids = request.data.get("metrics", []) + + if not isinstance(sources_ids, list): + return Response( + {"error": "metrics must be a list of IDs"}, status=status.HTTP_400_BAD_REQUEST + ) + + metric = self.service.get_metric(pk) + metric = self.service.add_sources(metric, sources_ids) + + return Response(MetricSerializer(metric).data, status=status.HTTP_200_OK) + + @extend_schema( + request=LinkSourceSerializer, + responses=MetricSerializer, + description="Removes a list of source IDs from an existing metric.", + tags=["Metrics"], + methods=["DELETE"], + ) + @action(detail=True, methods=["delete"], url_path="sources") + def remove_metrics(self, request, pk=None): + sources_ids = request.data.get("sources", []) + + if not isinstance(sources_ids, list): + return Response( + {"error": "metrics must be a list of IDs"}, status=status.HTTP_400_BAD_REQUEST + ) + + metric = self.service.get_metric(pk) + metric = self.service.remove_sources(metric, sources_ids) + + return Response(MetricSerializer(metric).data, status=status.HTTP_200_OK) + + +class MetricValueViewSet(viewsets.ViewSet): + service = MetricValueService() + + @extend_schema( + responses=MetricValueSerializer(many=True), + tags=["Metrics"], + ) + def list(self, request): + metrics = self.service.list_metric_values() + serializer = MetricValueSerializer(metrics, many=True) + return Response(serializer.data) + + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses=MetricValueSerializer, + tags=["Metrics"], + ) + def retrieve(self, request, pk=None): + metric = self.service.get_metric_value(pk) + serializer = MetricValueSerializer(metric) + return Response(serializer.data) diff --git a/metric/registry/__init__.py b/polling/__init__.py similarity index 100% rename from metric/registry/__init__.py rename to polling/__init__.py diff --git a/polling/admin.py b/polling/admin.py new file mode 100644 index 0000000..3fed5f0 --- /dev/null +++ b/polling/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin + +from polling.models import PollingSchedule + +admin.site.register(PollingSchedule) diff --git a/polling/apps.py b/polling/apps.py new file mode 100644 index 0000000..75ce417 --- /dev/null +++ b/polling/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class PollingConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "polling" diff --git a/polling/migrations/0001_initial.py b/polling/migrations/0001_initial.py new file mode 100644 index 0000000..9c5b805 --- /dev/null +++ b/polling/migrations/0001_initial.py @@ -0,0 +1,54 @@ +# Generated by Django 5.1.7 on 2025-05-04 19:10 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("app", "0003_alter_app_developer"), + ] + + operations = [ + migrations.CreateModel( + name="PollingSchedule", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("interval_hours", models.IntegerField()), + ("start_at", models.DateTimeField(auto_now_add=True)), + ("next_run", models.DateTimeField()), + ( + "poll_type", + models.CharField( + choices=[ + ("metrics", "Metrics only"), + ("reviews", "Reviews only"), + ("both", "Metrics + Reviews"), + ], + default="metrics", + max_length=10, + ), + ), + ("is_active", models.BooleanField(default=True)), + ( + "app", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="polling_schedules", + to="app.app", + ), + ), + ], + options={ + "unique_together": {("app", "poll_type")}, + }, + ), + ] diff --git a/polling/migrations/0002_pollingschedule_periodic_task.py b/polling/migrations/0002_pollingschedule_periodic_task.py new file mode 100644 index 0000000..4389c85 --- /dev/null +++ b/polling/migrations/0002_pollingschedule_periodic_task.py @@ -0,0 +1,26 @@ +# Generated by Django 5.1.7 on 2025-05-05 10:20 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("django_celery_beat", "0019_alter_periodictasks_options"), + ("polling", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="pollingschedule", + name="periodic_task", + field=models.OneToOneField( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="polling_schedule", + to="django_celery_beat.periodictask", + ), + ), + ] diff --git a/polling/migrations/__init__.py b/polling/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/polling/models.py b/polling/models.py new file mode 100644 index 0000000..7bbec20 --- /dev/null +++ b/polling/models.py @@ -0,0 +1,34 @@ +from django.db import models +from django_celery_beat.models import PeriodicTask + +from app.models import App + + +class PollingSchedule(models.Model): + app = models.ForeignKey(App, on_delete=models.CASCADE, related_name="polling_schedules") + interval_hours = models.IntegerField() + start_at = models.DateTimeField(auto_now_add=True) + next_run = models.DateTimeField() + + POLL_TYPE_CHOICES = [ + ("metrics", "Metrics only"), + ("reviews", "Reviews only"), + ("both", "Metrics + Reviews"), + ] + poll_type = models.CharField(max_length=10, choices=POLL_TYPE_CHOICES, default="metrics") + + is_active = models.BooleanField(default=True) + + periodic_task = models.OneToOneField( + PeriodicTask, + on_delete=models.CASCADE, + null=True, + blank=True, + related_name="polling_schedule", + ) + + class Meta: + unique_together = ("app", "poll_type") + + def __str__(self): + return f"{self.app.name} - {self.poll_type}" diff --git a/polling/repositories.py b/polling/repositories.py new file mode 100644 index 0000000..e4578a9 --- /dev/null +++ b/polling/repositories.py @@ -0,0 +1,71 @@ +import json + +from django_celery_beat.models import IntervalSchedule, PeriodicTask + +from .models import PollingSchedule + + +class PollingRepository: + @staticmethod + def get_all_active(poll_type): + queryset = PollingSchedule.objects.filter(is_active=True) + if poll_type: + queryset = queryset.filter(poll_type=poll_type) + return queryset + + @staticmethod + def get_by_app_id(app_id, poll_type): + try: + return PollingSchedule.objects.get(app_id=app_id, poll_type=poll_type) + except PollingSchedule.DoesNotExist: + return None + + @staticmethod + def create(app_id, interval_hours, poll_type, periodic_task): + from datetime import timedelta + + from django.utils.timezone import now + + start = now() + schedule = PollingSchedule.objects.create( + app_id=app_id, + interval_hours=interval_hours, + start_at=start, + next_run=start + timedelta(hours=interval_hours), + poll_type=poll_type, + is_active=True, + periodic_task=periodic_task, + ) + return schedule + + @staticmethod + def get_or_create_interval_schedule(interval_hours): + schedule, _ = IntervalSchedule.objects.get_or_create( + every=interval_hours, + period=IntervalSchedule.HOURS, + ) + return schedule + + @staticmethod + def create_periodic_task(schedule, app_id, poll_type, task): + name = f"{poll_type}-polling-{app_id}" + if PeriodicTask.objects.filter(name=name).exists(): + raise ValueError("PeriodicTask already exists with this name") + + task = PeriodicTask.objects.create( + interval=schedule, + name=name, + task=task, + args=json.dumps([app_id, poll_type]), + ) + return task + + @staticmethod + def delete(instance): + if instance.periodic_task: + instance.periodic_task.delete() + instance.delete() + + @staticmethod + def exists(app_id, poll_type): + return PollingSchedule.objects.filter(app_id=app_id, poll_type=poll_type).exists() diff --git a/polling/serializers.py b/polling/serializers.py new file mode 100644 index 0000000..5f53b8a --- /dev/null +++ b/polling/serializers.py @@ -0,0 +1,33 @@ +from django_celery_beat.models import PeriodicTask +from rest_framework import serializers + +from polling.models import PollingSchedule + + +class PeriodicTaskSerializer(serializers.ModelSerializer): + + class Meta: + model = PeriodicTask + fields = ["id", "name", "task", "args", "enabled", "last_run_at"] + + +class PollingScheduleSerializer(serializers.ModelSerializer): + periodic_task = PeriodicTaskSerializer(read_only=True) + + class Meta: + model = PollingSchedule + fields = [ + "app", + "interval_hours", + "start_at", + "next_run", + "is_active", + "periodic_task", + ] + read_only_fields = [ + "app", + "start_at", + "next_run", + "is_active", + "periodic_task", + ] diff --git a/polling/services.py b/polling/services.py new file mode 100644 index 0000000..51e320c --- /dev/null +++ b/polling/services.py @@ -0,0 +1,189 @@ +from datetime import datetime + +import requests +from django.utils.timezone import now + +from metric.models import Metric, MetricValue +from polling.tasks import run_polling_task +from review.repositories import ReviewRepository +from source.models import Source +from source.services import SourceService + +from .repositories import PollingRepository + + +class PollingService: + def __init__(self): + self.repo = PollingRepository() + self.source_service = SourceService() + + def _analyze_reviews(self, reviews, service_url: str, label: str = "RE-Miner"): + """ + Envia una llista de ressenyes a un servei RE-Miner i retorna la resposta. + + reviews: List[Dict] amb clau 'text' i opcionalment 'reviewId'. + service_url: URL del servei RE-Miner. + label: Nom del servei per mostrar en errors. + """ + try: + payload = { + "reviews": [ + {"reviewId": str(r.get("reviewId") or idx), "text": r["text"]} + for idx, r in enumerate(reviews) + ] + } + response = requests.post(service_url, json=payload, timeout=10) + response.raise_for_status() + return response.json() + except Exception as e: + print(f"[{label} ❌] Error analitzant ressenyes: {e}") + return [] + + def _analyze_review_polarity(self, reviews): + return self._analyze_reviews( + reviews, + service_url="http://re-miner-polarity:3010/analyze-polarity?polarity-service=MLP", + label="RE-Miner (Polarity)", + ) + + def _analyze_review_type(self, reviews): + return self._analyze_reviews( + reviews, + service_url="http://re-miner-type:3011/analyze-type?type-service=DISTILBERT", + label="RE-Miner (Type)", + ) + + def list_active_polling_schedules(self, poll_type=None): + return self.repo.get_all_active(poll_type) + + def get_polling_schedule(self, app_id, poll_type=None): + return self.repo.get_by_app_id(app_id, poll_type) + + def create_polling_schedule(self, app_id, interval_hours=None, poll_type=None): + if self.repo.exists(app_id, poll_type): + raise ValueError(f"Polling already scheduled for app {app_id} and type {poll_type}") + + interval_schedule = self.repo.get_or_create_interval_schedule(interval_hours) + periodic_task = self.repo.create_periodic_task( + interval_schedule, app_id, poll_type, "polling.tasks.run_polling_task" + ) + schedule = self.repo.create(app_id, interval_hours, poll_type, periodic_task) + + run_polling_task.delay(app_id, poll_type) + + return schedule + + def delete_schedule(self, instance): + return self.repo.delete(instance) + + def poll_reviews(self, app_id, date_from=None, date_to=None): + adapters = self.source_service.load_sources() + results = [] + + for adapter in adapters: + reviews = adapter.fetch_reviews(app_id=app_id, date_from=date_from, date_to=date_to) + if not reviews: + results.append( + { + "adapter": adapter.name, + "fetched": 0, + "saved": 0, + "reviews": [], + } + ) + continue + + # 🧪 Preparem les dades per RE-Miner (agafa 'content' com a 'text') + for idx, r in enumerate(reviews): + r["reviewId"] = r.get("reviewId", str(idx)) # assegura que sempre hi ha un id + + prepared_reviews = [{"reviewId": r["reviewId"], "text": r["content"]} for r in reviews] + polarity_result = self._analyze_review_polarity(prepared_reviews) + type_result = self._analyze_review_type(prepared_reviews) + + # 🔁 Mapegem la polaritat a cada review original + polarity_by_id = { + r["reviewId"]: r["polarity"] for r in polarity_result.get("reviews", []) + } + + type_by_id = {r["reviewId"]: r["type"] for r in type_result.get("reviews", [])} + + for review in reviews: + review["polarity"] = polarity_by_id.get(review["reviewId"]) + review["type"] = type_by_id.get(review["reviewId"]) + + saved_count = ReviewRepository().save_reviews(app_id, reviews) + + results.append( + { + "adapter": adapter.name, + "fetched": len(reviews), + "saved": saved_count, + "reviews": reviews, + } + ) + + return results + + def poll_metrics(self, app_id): + adapters = self.source_service.load_sources() + print("[🔁] Iniciant recollida de mètriques...") + + metrics = list(Metric.objects.prefetch_related("sources").all()) + + # 🔀 Separa mètriques + external_metrics = [m for m in metrics if not m.is_internal] + internal_metrics = [m for m in metrics if m.is_internal] + + # ⚡ Caches + metric_cache = {m.code: m for m in metrics} + source_cache = {a.code: Source.objects.get(code=a.code) for a in adapters} + + # 🔁 EXTERNAL: Per adapters + for adapter in adapters: + print("adapter actual: ", adapter.code) + source_code = adapter.code + available_metrics = [ + m.code + for m in external_metrics + if source_code in m.sources.values_list("code", flat=True) + ] + if not available_metrics: + continue + + try: + result = adapter.fetch(app_id, available_metrics) + except Exception as e: + print(f"❌ Error en adapter `{source_code}`: {e}") + continue + + for metric_code, value in result.items(): + if value in [None, ""]: + print(f" ⚠️ {metric_code} no ha retornat valor.") + continue + + metric = metric_cache[metric_code] + source = source_cache[source_code] + + already_exists = MetricValue.objects.filter( + app_id=app_id, metric=metric, source=source, retrieved_at__date=now().date() + ).exists() + + if already_exists: + print(f" ℹ️ Ja existia: {metric_code} — {value}") + continue + + MetricValue.objects.create( + app_id=app_id, + metric=metric, + source=source, + value=value, + retrieved_at=datetime.now(), + ) + print(f" ✅ Guardat {metric_code}: {value}") + + # 🔁 INTERNAL: opcional (implementa si tens estratègies internes) + if internal_metrics: + print("\n📦 Tractament de mètriques internes no implementat (pendents).") + + print("\n✅ Procés de recollida completat.") diff --git a/polling/tasks.py b/polling/tasks.py new file mode 100644 index 0000000..a0d3619 --- /dev/null +++ b/polling/tasks.py @@ -0,0 +1,27 @@ +import time + +from celery import shared_task + + +@shared_task +def run_polling_task(app_id, poll_type): + from polling.services import PollingService + + service = PollingService() + + if poll_type == "metrics": + print("poll_metrics completat") + return service.poll_metrics(app_id) + elif poll_type == "reviews": + print("poll_reviews completat") + return service.poll_reviews(app_id) + else: + raise ValueError(f"Unknown poll_type: {poll_type}") + + +@shared_task +def prueba_celery(): + print("Executant tasca de prova...") + time.sleep(5) + print("Tasca completada correctament!") + return "Fet!" diff --git a/polling/tests.py b/polling/tests.py new file mode 100644 index 0000000..e69de29 diff --git a/polling/urls.py b/polling/urls.py new file mode 100644 index 0000000..0a05ddd --- /dev/null +++ b/polling/urls.py @@ -0,0 +1,18 @@ +from django.urls import path + +from polling.views import PollingScheduleViewSet + +polling_retrieve_view = PollingScheduleViewSet.as_view({"get": "retrieve"}) +polling_list_view = PollingScheduleViewSet.as_view({"get": "list"}) +polling_detail_view = PollingScheduleViewSet.as_view( + {"get": "retrieve", "post": "create", "delete": "destroy"} +) +polling_review_view = PollingScheduleViewSet.as_view({"post": "poll_reviews"}) + +urlpatterns = [ + path("metrics/", polling_list_view, {"poll_type": "metrics"}), + path("reviews/", polling_list_view, {"poll_type": "reviews"}), + path("metrics/apps//", polling_detail_view, {"poll_type": "metrics"}), + path("reviews/apps//", polling_detail_view, {"poll_type": "reviews"}), + path("maunal/reviews/apps//", polling_review_view), +] diff --git a/polling/views.py b/polling/views.py new file mode 100644 index 0000000..a5b6bba --- /dev/null +++ b/polling/views.py @@ -0,0 +1,105 @@ +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_schema +from rest_framework import status, viewsets +from rest_framework.response import Response + +from polling.serializers import PollingScheduleSerializer +from polling.services import PollingService + + +class PollingScheduleViewSet(viewsets.ViewSet): + service = PollingService() + + @extend_schema( + description="List all active metric polling schedules.", + responses={200: PollingScheduleSerializer(many=True)}, + tags=["Polling"], + ) + def list(self, request, poll_type=None): + schedules = self.service.list_active_polling_schedules(poll_type) + serializer = PollingScheduleSerializer(schedules, many=True) + return Response(serializer.data, status=status.HTTP_200_OK) + + @extend_schema( + description="Check if there is an active polling schedule for an app.", + parameters=[ + OpenApiParameter( + name="id", required=True, type=OpenApiTypes.INT, location=OpenApiParameter.PATH + ), + ], + responses={200: PollingScheduleSerializer}, + tags=["Polling"], + ) + def retrieve(self, request, id=None, poll_type=None): + schedule = self.service.get_polling_schedule(id, poll_type) + serializer = PollingScheduleSerializer(schedule) + return Response(serializer.data, status=status.HTTP_200_OK) + + @extend_schema( + request=PollingScheduleSerializer, + responses={201: PollingScheduleSerializer}, + tags=["Polling"], + ) + def create(self, request, id=None, poll_type=None): + serializer = PollingScheduleSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + interval = serializer.validated_data["interval_hours"] + + try: + schedule = self.service.create_polling_schedule(id, interval, poll_type) + except ValueError as e: + return Response({"detail": str(e)}, status=409) + + return Response(PollingScheduleSerializer(schedule).data, status=201) + + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses={204: None}, + tags=["Polling"], + ) + def destroy(self, request, id=None, poll_type=None): + schedule = self.service.get_polling_schedule(id, poll_type) + self.service.delete_schedule(schedule) + return Response(status=status.HTTP_204_NO_CONTENT) + + @extend_schema( + request=None, + parameters=[ + OpenApiParameter( + name="id", + type=int, + required=True, + location="path", + description="App ID to fetch reviews for", + ), + OpenApiParameter( + name="date_from", type=str, description="Start date (YYYY-MM-DD)", required=False + ), + OpenApiParameter( + name="date_to", type=str, description="End date (YYYY-MM-DD)", required=False + ), + ], + responses={ + 200: OpenApiResponse(description="Reviews successfully fetched and saved."), + 400: OpenApiResponse( + description="Invalid request. Possible causes: missing parameters, " + "wrong date format, or invalid date range." + ), + 404: OpenApiResponse(description="App with the given ID was not found."), + }, + tags=["Polling"], + description=( + "Polls Google Play for reviews of a given app between two dates.\n\n" + "Saves new ones in the database and returns" + "the fetched reviews with stats.\n\n" + "If no `date_from` and `date_to` are provided," + " reviews from **yesterday** will be fetched." + ), + ) + def poll_reviews(self, request, id=None): + date_from = request.query_params.get("date_from") + date_to = request.query_params.get("date_to") + + result = self.service.poll_reviews(app_id=id, date_from=date_from, date_to=date_to) + + return Response(result) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c8d1fe8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,11 @@ +[tool.black] +line-length = 100 +target-version = ['py312'] + +[tool.isort] +profile = "black" +line_length = 100 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..7a4fb9b --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +DJANGO_SETTINGS_MODULE = config.settings +python_files = tests.py test_*.py *_tests.py diff --git a/requirements.txt b/requirements.txt index fa61143..2b4228e 100644 Binary files a/requirements.txt and b/requirements.txt differ diff --git a/review/migrations/0001_initial.py b/review/migrations/0001_initial.py index a9d3f5a..fcecd63 100644 --- a/review/migrations/0001_initial.py +++ b/review/migrations/0001_initial.py @@ -83,9 +83,7 @@ class Migration(migrations.Migration): ("date", models.DateTimeField()), ( "app", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, to="app.app" - ), + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="app.app"), ), ], ), diff --git a/review/migrations/0002_review_app_version_review_replied_at_and_more.py b/review/migrations/0002_review_app_version_review_replied_at_and_more.py new file mode 100644 index 0000000..ab7e30a --- /dev/null +++ b/review/migrations/0002_review_app_version_review_replied_at_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 5.1.7 on 2025-04-29 09:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("review", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="review", + name="app_version", + field=models.CharField(blank=True, max_length=100, null=True), + ), + migrations.AddField( + model_name="review", + name="replied_at", + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name="review", + name="reply_content", + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name="review", + name="review_id", + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/review/models.py b/review/models.py index 9077805..6fef8be 100644 --- a/review/models.py +++ b/review/models.py @@ -12,8 +12,9 @@ class ReviewPolarity(models.TextChoices): class ReviewType(models.TextChoices): BUG = "bug", "Bug" + RATING = "rating", "Rating" FEATURE = "feature", "Feature" - GENERAL = "general", "General" + USER_EXPERIENCE = "user_experience", "User Experience" class ReviewTopic(models.TextChoices): @@ -23,12 +24,17 @@ class ReviewTopic(models.TextChoices): EFFECTIVENESS = "effectiveness", "Effectiveness" EFFICIENCY = "efficiency", "Efficiency" ENJOYABILITY = "enjoyability", "Enjoyability" + GENERAL = "general", "General" LEARNABILITY = "learnability", "Learnability" + N_A = "n/a", "N/A" + RELIABILITY = "reliability", "Reliability" + SAFETY = "safety", "Safety" SECURITY = "security", "Security" USABILITY = "usability", "Usability" class Review(models.Model): + review_id = models.CharField(max_length=255, null=True, blank=True) app = models.ForeignKey(App, on_delete=models.CASCADE) author = models.CharField(max_length=100, null=True, blank=True) content = models.TextField() @@ -37,10 +43,11 @@ class Review(models.Model): ) polarity = models.CharField(max_length=10, choices=ReviewPolarity.choices) type = models.CharField(max_length=20, choices=ReviewType.choices) - topic = models.CharField( - max_length=30, choices=ReviewTopic.choices, null=True, blank=True - ) + topic = models.CharField(max_length=30, choices=ReviewTopic.choices, null=True, blank=True) date = models.DateTimeField() + reply_content = models.TextField(null=True, blank=True) + replied_at = models.DateTimeField(null=True, blank=True) + app_version = models.CharField(max_length=100, null=True, blank=True) def __str__(self): return f"{self.app.name} - {self.date.date()} - {self.author or 'Anon'}" diff --git a/review/repositories.py b/review/repositories.py new file mode 100644 index 0000000..560cce6 --- /dev/null +++ b/review/repositories.py @@ -0,0 +1,82 @@ +from datetime import datetime + +from django.utils import timezone + +from .models import Review + + +class ReviewRepository: + def get_all(self, filters=None): + queryset = Review.objects.all() + + if filters: + app_id = filters.get("app") + date_from = filters.get("date_from") + date_to = filters.get("date_to") + + if app_id: + queryset = queryset.filter(app_id=app_id) + + if date_from: + try: + queryset = queryset.filter(date__gte=datetime.fromisoformat(date_from)) + except ValueError: + pass + + if date_to: + try: + queryset = queryset.filter(date__lte=datetime.fromisoformat(date_to)) + except ValueError: + pass + + return queryset + + def get_by_id(self, review_id): + return Review.objects.get(id=review_id) + + def save_reviews(self, app_id, reviews_list): + if not reviews_list: + return 0 + + # Agafar tots els review_ids existents d'aquesta app + existing_review_ids = set( + Review.objects.filter( + review_id__in=[r.get("reviewId") for r in reviews_list] + ).values_list("review_id", flat=True) + ) + + created_reviews = [] + + for r in reviews_list: + review_id = r.get("reviewId") + if not review_id or review_id in existing_review_ids: + continue + + created_at = r.get("at") + if created_at and timezone.is_naive(created_at): + created_at = timezone.make_aware(created_at) + + replied_at = r.get("repliedAt") + if replied_at and timezone.is_naive(replied_at): + replied_at = timezone.make_aware(replied_at) + + review = Review( + app_id=app_id, + review_id=review_id, + author=r.get("userName"), + content=r.get("content"), + rating=r.get("score"), + # polarity=self._infer_polarity(r.get('score')), + # type="general", + # topic=None, + date=created_at, + reply_content=r.get("replyContent"), + replied_at=replied_at, + app_version=r.get("appVersion"), + ) + created_reviews.append(review) + + Review.objects.bulk_create(created_reviews) + print(f"Saved {len(created_reviews)} new reviews.") + + return len(created_reviews) diff --git a/review/serializers.py b/review/serializers.py new file mode 100644 index 0000000..0a84f20 --- /dev/null +++ b/review/serializers.py @@ -0,0 +1,42 @@ +from rest_framework import serializers + +from .models import Review, ReviewPolarity, ReviewTopic, ReviewType + + +class ReviewSerializer(serializers.ModelSerializer): + polarity = serializers.ChoiceField(choices=ReviewPolarity.choices) + type = serializers.ChoiceField(choices=ReviewType.choices) + topic = serializers.ChoiceField(choices=ReviewTopic.choices) + + class Meta: + model = Review + fields = [ + "id", + "review_id", + "app", + "author", + "content", + "rating", + "polarity", + "type", + "topic", + "date", + "reply_content", + "replied_at", + "app_version", + ] + read_only_fields = [ + "id", + "review_id", + "app", + "author", + "content", + "rating", + "polarity", + "type", + "topic", + "date", + "reply_content", + "replied_at", + "app_version", + ] diff --git a/review/services.py b/review/services.py new file mode 100644 index 0000000..7c23b28 --- /dev/null +++ b/review/services.py @@ -0,0 +1,18 @@ +from django.core.exceptions import ObjectDoesNotExist +from rest_framework.exceptions import NotFound + +from .repositories import ReviewRepository + + +class ReviewService: + def __init__(self): + self.repo = ReviewRepository() + + def list_reviews(self, filters=None): + return self.repo.get_all(filters=filters) + + def get_review(self, review_id): + try: + return self.repo.get_by_id(review_id) + except ObjectDoesNotExist: + raise NotFound(f"The review with ID '{review_id}' is not registered.") diff --git a/review/tests.py b/review/tests.py index e69de29..9f81184 100644 --- a/review/tests.py +++ b/review/tests.py @@ -0,0 +1,53 @@ +import pytest +from django.utils import timezone + +from app.models import App +from review.models import Review, ReviewPolarity, ReviewTopic, ReviewType + + +@pytest.mark.django_db +def test_create_review_no_values(): + now = timezone.now() + a = App.objects.create( + code="test_app", + name="Test App", + description="Descripction of the test app", + appstore_id="123456", + ) + r = Review.objects.create( + app=a, + content="Content of the test review", + date=now, + ) + assert r.pk is not None + assert r.app == a + assert r.content == "Content of the test review" + assert r.date == now + + +@pytest.mark.django_db +def test_create_review_with_values(): + now = timezone.now() + a = App.objects.create( + code="test_app", + name="Test App", + description="Descripction of the test app", + appstore_id="123456", + ) + r = Review.objects.create( + app=a, + content="Content of the test review", + date=now, + rating=5, + polarity=ReviewPolarity.NEUTRAL, + type=ReviewType.FEATURE, + topic=ReviewTopic.COMPATIBILITY, + ) + assert r.pk is not None + assert r.app == a + assert r.content == "Content of the test review" + assert r.date == now + assert r.rating == 5 + assert r.polarity == ReviewPolarity.NEUTRAL + assert r.type == ReviewType.FEATURE + assert r.topic == ReviewTopic.COMPATIBILITY diff --git a/review/urls.py b/review/urls.py new file mode 100644 index 0000000..f43b970 --- /dev/null +++ b/review/urls.py @@ -0,0 +1,11 @@ +from django.urls import include, path +from rest_framework.routers import DefaultRouter + +from .views import ReviewViewSet + +router = DefaultRouter() +router.register(r"reviews", ReviewViewSet, basename="review") + +urlpatterns = [ + path("", include(router.urls)), +] diff --git a/review/views.py b/review/views.py index e69de29..006e335 100644 --- a/review/views.py +++ b/review/views.py @@ -0,0 +1,39 @@ +from drf_spectacular.utils import OpenApiParameter, extend_schema +from rest_framework import viewsets +from rest_framework.response import Response + +from .serializers import ReviewSerializer +from .services import ReviewService + + +class ReviewViewSet(viewsets.ViewSet): + service = ReviewService() + + @extend_schema( + parameters=[ + OpenApiParameter(name="app", type=int, description="Filter by app ID"), + OpenApiParameter(name="date_from", type=str, description="Start date (YYYY-MM-DD)"), + OpenApiParameter(name="date_to", type=str, description="End date (YYYY-MM-DD)"), + ], + responses=ReviewSerializer(many=True), + tags=["Reviews"], + ) + def list(self, request): + filters = { + "app": request.query_params.get("app"), + "date_from": request.query_params.get("date_from"), + "date_to": request.query_params.get("date_to"), + } + reviews = self.service.list_reviews(filters) + serializer = ReviewSerializer(reviews, many=True) + return Response(serializer.data) + + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses=ReviewSerializer, + tags=["Reviews"], + ) + def retrieve(self, request, pk=None): + source = self.service.get_review(pk) + serializer = ReviewSerializer(source) + return Response(serializer.data) diff --git a/schema.yaml b/schema.yaml new file mode 100644 index 0000000..0fb9c0b --- /dev/null +++ b/schema.yaml @@ -0,0 +1,1237 @@ +openapi: 3.0.3 +info: + title: '' + version: 0.0.0 +paths: + /api/apps/: + get: + operationId: api_apps_list + tags: + - Apps + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/App' + description: '' + post: + operationId: api_apps_create + description: |- + Creates a new app. + + The fields 'available_on_ios', 'available_on_android', 'pegi_rating', 'release_date', and 'min_ios_version' will be automatically populated by fetching data from the App Store and/or Google Play APIs based on the provided identifiers. + + You do not need to manually provide these fields when creating an app. + tags: + - Apps + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/AppCreate' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/AppCreate' + multipart/form-data: + schema: + $ref: '#/components/schemas/AppCreate' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '201': + content: + application/json: + schema: + $ref: '#/components/schemas/App' + description: '' + /api/apps/{id}/: + get: + operationId: api_apps_retrieve + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Apps + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/App' + description: '' + put: + operationId: api_apps_update + parameters: + - in: path + name: id + schema: + type: string + required: true + tags: + - Apps + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/App' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/App' + multipart/form-data: + schema: + $ref: '#/components/schemas/App' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/App' + description: '' + delete: + operationId: api_apps_destroy + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Apps + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '204': + description: No response body + /api/metric-values/: + get: + operationId: api_metric_values_list + tags: + - Metrics + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/MetricValue' + description: '' + /api/metric-values/{id}/: + get: + operationId: api_metric_values_retrieve + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Metrics + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/MetricValue' + description: '' + /api/metrics/: + get: + operationId: api_metrics_list + tags: + - Metrics + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Metric' + description: '' + post: + operationId: api_metrics_create + description: |- + Creates a new data metric. + + The 'source' field is read-only and cannot be set directly on creation. + tags: + - Metrics + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Metric' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/Metric' + multipart/form-data: + schema: + $ref: '#/components/schemas/Metric' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '201': + content: + application/json: + schema: + $ref: '#/components/schemas/Metric' + description: '' + /api/metrics/{id}/: + get: + operationId: api_metrics_retrieve + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Metrics + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Metric' + description: '' + put: + operationId: api_metrics_update + parameters: + - in: path + name: id + schema: + type: string + required: true + tags: + - Metrics + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Metric' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/Metric' + multipart/form-data: + schema: + $ref: '#/components/schemas/Metric' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Metric' + description: '' + delete: + operationId: api_metrics_destroy + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Metrics + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '204': + description: No response body + /api/metrics/{id}/sources/: + post: + operationId: api_metrics_sources_create + description: Adds a list of source IDs to an existing metric without removing + existing ones. + parameters: + - in: path + name: id + schema: + type: string + required: true + tags: + - Metrics + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/LinkSource' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/LinkSource' + multipart/form-data: + schema: + $ref: '#/components/schemas/LinkSource' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Metric' + description: '' + delete: + operationId: api_metrics_sources_destroy + description: Removes a list of source IDs from an existing metric. + parameters: + - in: path + name: id + schema: + type: string + required: true + tags: + - Metrics + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '204': + description: No response body + /api/polling/metrics/: + get: + operationId: api_polling_metrics_list + description: List all active metric polling schedules. + tags: + - Polling + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/PollingSchedule' + description: '' + /api/polling/metrics/apps/{id}/: + get: + operationId: api_polling_metrics_apps_retrieve + description: Check if there is an active polling schedule for an app. + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Polling + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/PollingSchedule' + description: '' + post: + operationId: api_polling_metrics_apps_create + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - api + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PollingSchedule' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/PollingSchedule' + multipart/form-data: + schema: + $ref: '#/components/schemas/PollingSchedule' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '201': + content: + application/json: + schema: + $ref: '#/components/schemas/PollingSchedule' + description: '' + /api/polling/reviews/: + get: + operationId: api_polling_reviews_list + description: List all active metric polling schedules. + tags: + - Polling + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/PollingSchedule' + description: '' + /api/polling/reviews/apps/{id}/: + get: + operationId: api_polling_reviews_apps_retrieve + description: Check if there is an active polling schedule for an app. + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Polling + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/PollingSchedule' + description: '' + post: + operationId: api_polling_reviews_apps_create + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - api + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PollingSchedule' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/PollingSchedule' + multipart/form-data: + schema: + $ref: '#/components/schemas/PollingSchedule' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '201': + content: + application/json: + schema: + $ref: '#/components/schemas/PollingSchedule' + description: '' + /api/reviews/: + get: + operationId: api_reviews_list + parameters: + - in: query + name: app + schema: + type: integer + description: Filter by app ID + - in: query + name: date_from + schema: + type: string + description: Start date (YYYY-MM-DD) + - in: query + name: date_to + schema: + type: string + description: End date (YYYY-MM-DD) + tags: + - Reviews + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Review' + description: '' + /api/reviews/{id}/: + get: + operationId: api_reviews_retrieve + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Reviews + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Review' + description: '' + /api/schema/: + get: + operationId: api_schema_retrieve + description: |- + OpenApi3 schema for this API. Format can be selected via content negotiation. + + - YAML: application/vnd.oai.openapi + - JSON: application/vnd.oai.openapi+json + parameters: + - in: query + name: format + schema: + type: string + enum: + - json + - yaml + - in: query + name: lang + schema: + type: string + enum: + - af + - ar + - ar-dz + - ast + - az + - be + - bg + - bn + - br + - bs + - ca + - ckb + - cs + - cy + - da + - de + - dsb + - el + - en + - en-au + - en-gb + - eo + - es + - es-ar + - es-co + - es-mx + - es-ni + - es-ve + - et + - eu + - fa + - fi + - fr + - fy + - ga + - gd + - gl + - he + - hi + - hr + - hsb + - hu + - hy + - ia + - id + - ig + - io + - is + - it + - ja + - ka + - kab + - kk + - km + - kn + - ko + - ky + - lb + - lt + - lv + - mk + - ml + - mn + - mr + - ms + - my + - nb + - ne + - nl + - nn + - os + - pa + - pl + - pt + - pt-br + - ro + - ru + - sk + - sl + - sq + - sr + - sr-latn + - sv + - sw + - ta + - te + - tg + - th + - tk + - tr + - tt + - udm + - ug + - uk + - ur + - uz + - vi + - zh-hans + - zh-hant + tags: + - api + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/vnd.oai.openapi: + schema: + type: object + additionalProperties: {} + application/yaml: + schema: + type: object + additionalProperties: {} + application/vnd.oai.openapi+json: + schema: + type: object + additionalProperties: {} + application/json: + schema: + type: object + additionalProperties: {} + description: '' + /api/sources/: + get: + operationId: api_sources_list + tags: + - Sources + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Source' + description: '' + post: + operationId: api_sources_create + description: |- + Creates a new data source. + + The 'metrics' field is read-only and cannot be set directly on creation. + tags: + - Sources + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Source' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/Source' + multipart/form-data: + schema: + $ref: '#/components/schemas/Source' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '201': + content: + application/json: + schema: + $ref: '#/components/schemas/Source' + description: '' + /api/sources/{id}/: + get: + operationId: api_sources_retrieve + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Sources + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Source' + description: '' + put: + operationId: api_sources_update + parameters: + - in: path + name: id + schema: + type: string + required: true + tags: + - Sources + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Source' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/Source' + multipart/form-data: + schema: + $ref: '#/components/schemas/Source' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Source' + description: '' + delete: + operationId: api_sources_destroy + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Sources + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '204': + description: No response body + /api/sources/{id}/metrics/: + post: + operationId: api_sources_metrics_create + description: Adds a list of metric IDs to an existing source without removing + existing ones. + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Sources + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/LinkMetrics' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/LinkMetrics' + multipart/form-data: + schema: + $ref: '#/components/schemas/LinkMetrics' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Source' + description: '' + delete: + operationId: api_sources_metrics_destroy + description: Removes a list of metric IDs from an existing source. + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Sources + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '204': + description: No response body +components: + schemas: + App: + type: object + properties: + id: + type: integer + readOnly: true + code: + type: string + maxLength: 100 + name: + type: string + maxLength: 200 + description: + type: string + appstore_id: + type: string + nullable: true + maxLength: 100 + playstore_id: + type: string + nullable: true + maxLength: 100 + developer: + type: string + nullable: true + maxLength: 100 + available_on_ios: + type: boolean + available_on_android: + type: boolean + pegi_rating: + type: string + nullable: true + maxLength: 10 + release_date: + type: string + format: date + nullable: true + min_ios_version: + type: string + nullable: true + maxLength: 10 + required: + - code + - description + - id + - name + AppCreate: + type: object + properties: + code: + type: string + maxLength: 100 + name: + type: string + maxLength: 200 + description: + type: string + appstore_id: + type: string + nullable: true + maxLength: 100 + playstore_id: + type: string + nullable: true + maxLength: 100 + required: + - code + - description + - name + LinkMetrics: + type: object + properties: + metrics: + type: array + items: + type: integer + description: List of metric IDs to link to the source. + required: + - metrics + LinkSource: + type: object + properties: + sources: + type: array + items: + type: integer + description: List of source IDs to link to the metric. + required: + - sources + Metric: + type: object + properties: + id: + type: integer + readOnly: true + code: + type: string + maxLength: 100 + name: + type: string + maxLength: 100 + description: + type: string + value_type: + $ref: '#/components/schemas/ValueTypeEnum' + sources: + type: array + items: + type: integer + readOnly: true + required: + - code + - description + - id + - name + - sources + - value_type + MetricValue: + type: object + properties: + id: + type: integer + readOnly: true + app: + type: integer + readOnly: true + metric: + type: integer + readOnly: true + source: + type: integer + readOnly: true + nullable: true + value: + type: string + readOnly: true + retrieved_at: + type: string + format: date-time + readOnly: true + required: + - app + - id + - metric + - retrieved_at + - source + - value + PolarityEnum: + enum: + - positive + - neutral + - negative + type: string + description: |- + * `positive` - Positive + * `neutral` - Neutral + * `negative` - Negative + PollTypeEnum: + enum: + - metrics + - reviews + - both + type: string + description: |- + * `metrics` - Metrics only + * `reviews` - Reviews only + * `both` - Metrics + Reviews + PollingSchedule: + type: object + properties: + app: + type: integer + interval_hours: + type: integer + maximum: 9223372036854775807 + minimum: -9223372036854775808 + format: int64 + start_at: + type: string + format: date-time + readOnly: true + next_run: + type: string + format: date-time + readOnly: true + poll_type: + allOf: + - $ref: '#/components/schemas/PollTypeEnum' + default: metrics + is_active: + type: boolean + readOnly: true + required: + - app + - interval_hours + - is_active + - next_run + - start_at + Review: + type: object + properties: + id: + type: integer + readOnly: true + review_id: + type: string + readOnly: true + nullable: true + app: + type: integer + readOnly: true + author: + type: string + readOnly: true + nullable: true + content: + type: string + readOnly: true + rating: + type: number + format: double + readOnly: true + nullable: true + polarity: + $ref: '#/components/schemas/PolarityEnum' + type: + $ref: '#/components/schemas/ReviewTypeEnum' + topic: + $ref: '#/components/schemas/TopicEnum' + date: + type: string + format: date-time + readOnly: true + reply_content: + type: string + readOnly: true + nullable: true + replied_at: + type: string + format: date-time + readOnly: true + nullable: true + app_version: + type: string + readOnly: true + nullable: true + required: + - app + - app_version + - author + - content + - date + - id + - polarity + - rating + - replied_at + - reply_content + - review_id + - topic + - type + ReviewTypeEnum: + enum: + - bug + - feature + - general + type: string + description: |- + * `bug` - Bug + * `feature` - Feature + * `general` - General + Source: + type: object + properties: + id: + type: integer + readOnly: true + code: + type: string + maxLength: 100 + name: + type: string + maxLength: 100 + type: + allOf: + - $ref: '#/components/schemas/SourceTypeEnum' + description: |- + Type of the source. Available options: + - 'api': Data source accessible through a public API. + - 'scraper': Data source obtained via web scraping. + - 'external_tool': Data provided by an external tool. + + * `api` - API + * `scraper` - Scraper + * `external_tool` - External Tool + url: + type: string + format: uri + nullable: true + maxLength: 200 + metrics: + type: array + items: + type: integer + readOnly: true + required: + - code + - id + - metrics + - name + - type + SourceTypeEnum: + enum: + - api + - scraper + - external_tool + type: string + description: |- + * `api` - API + * `scraper` - Scraper + * `external_tool` - External Tool + TopicEnum: + enum: + - aesthetics + - compatibility + - cost + - effectiveness + - efficiency + - enjoyability + - learnability + - security + - usability + type: string + description: |- + * `aesthetics` - Aesthetics + * `compatibility` - Compatibility + * `cost` - Cost + * `effectiveness` - Effectiveness + * `efficiency` - Efficiency + * `enjoyability` - Enjoyability + * `learnability` - Learnability + * `security` - Security + * `usability` - Usability + ValueTypeEnum: + enum: + - string + - integer + - float + - date + - boolean + type: string + description: |- + * `string` - String + * `integer` - Integer + * `float` - Float + * `date` - Date + * `boolean` - Boolean + securitySchemes: + basicAuth: + type: http + scheme: basic + cookieAuth: + type: apiKey + in: cookie + name: sessionid diff --git a/source/adapters/base.py b/source/adapters/base.py index e1611a9..524dbba 100644 --- a/source/adapters/base.py +++ b/source/adapters/base.py @@ -1,7 +1,26 @@ -from typing import Protocol +from abc import ABC, abstractmethod +from typing import Optional +from source.constants.source_type import SourceType -class SourceAdapter(Protocol): + +class SourceAdapter(ABC): + code: str + name: str + type: SourceType + url: Optional[str] + supported_metrics: list[str] + api_key: Optional[str] = None + + def __init__(self): + if not hasattr(self, "code") or not self.code: + raise NotImplementedError("Adapter must define 'code' and load source config.") + + @abstractmethod def supports_metric(self, metric: str) -> bool: ... - def fetch(self, app_name: str, metric: str) -> str: ... + @abstractmethod + def fetch(self, app_id: int, metrics: list[str]) -> dict[str, str]: ... + + def fetch_reviews(self, app_id: int, date_from=None, date_to=None): + return [] diff --git a/source/adapters/google_play_scraper.py b/source/adapters/google_play_scraper.py new file mode 100644 index 0000000..ce0ba85 --- /dev/null +++ b/source/adapters/google_play_scraper.py @@ -0,0 +1,107 @@ +import time +from datetime import date, datetime, timedelta + +from google_play_scraper import app as gp_app +from google_play_scraper import reviews +from rest_framework.exceptions import ParseError, ValidationError + +from app.services import AppService +from metric.constants import MetricCodes +from source.adapters.base import SourceAdapter +from source.services import SourceService + + +class GooglePlayScraperAdapter(SourceAdapter): + code = "google_play" + + def __init__(self): + super().__init__() + source_data = SourceService().get_source_data(code=self.code) + self.name = source_data["name"] + self.type = source_data["type"] + self.url = source_data["url"] + self.supported_metrics = source_data["supported_metrics"] + + def supports_metric(self, metric: str): + return metric in self.supported_metrics + + @staticmethod + def lookup_app(package_name: str) -> dict | None: + try: + result = gp_app(package_name, lang="en", country="uk") + return result + except Exception as e: + print(f"[GooglePlayScraperAdapter] Error fetching app data: {e}") + return None + + def fetch(self, app_id: int, metrics: list[str]): + app = AppService().get_app(app_id=app_id) + if not app: + return {} + + result = self.lookup_app(app.playstore_id) + + metric_map = { + MetricCodes.AVERAGE_RATING: str(result.get("score") or ""), + MetricCodes.TOTAL_REVIEWS: str(result.get("reviews") or ""), + MetricCodes.TOTAL_DOWNLOADS: str(result.get("realInstalls") or ""), + MetricCodes.LAST_UPDATE_DATE: str(result.get("lastUpdatedOn") or ""), + } + + return { + metric: metric_map.get(metric, "") for metric in metrics if self.supports_metric(metric) + } + + def fetch_reviews(self, app_id: int, date_from=None, date_to=None): + app = AppService().get_app(app_id=app_id) + if not app: + return [] + + # Dates per defecte: ahir + yesterday = date.today() - timedelta(days=1) + try: + date_from = datetime.fromisoformat(date_from).date() if date_from else yesterday + date_to = datetime.fromisoformat(date_to).date() if date_to else yesterday + except ValueError: + raise ParseError("Dates must be in ISO format: YYYY-MM-DD.") + + days_range = (date_to - date_from).days + if days_range < 0: + raise ValidationError("date_from must be before or equal to date_to.") + + count_per_request = min(max(1, 180 * days_range), 4500) + + all_reviews = [] + next_token = None + count = 0 + + while True: + batch, next_token = reviews( + app.playstore_id, + lang="en", + country="us", + count=count_per_request, + continuation_token=next_token, + ) + count += 1 + + if not batch: + break + + for r in batch: + review_date = r["at"].date() + + if review_date < date_from: + print("⛔ Hem arribat a reviews més antigues del rang. Sortint.") + return all_reviews + + if review_date <= date_to: + all_reviews.append(r) + + if not next_token: + break + + time.sleep(1) + + print(f"✅ Total reviews: {len(all_reviews)} recollides en {count} peticions.") + return all_reviews diff --git a/source/adapters/itunes.py b/source/adapters/itunes.py index d4675c9..7226016 100644 --- a/source/adapters/itunes.py +++ b/source/adapters/itunes.py @@ -1,13 +1,48 @@ +import requests + +from app.services import AppService +from metric.constants import MetricCodes from source.adapters.base import SourceAdapter +from source.services import SourceService class ItunesSearchAPIAdapter(SourceAdapter): - def supports_metric(self, metric: str) -> bool: - return metric in ["average_rating", "total_reviews"] - - def fetch(self, app_id: str, metric: str): - if metric == "average_rating": - return 4.5 - elif metric == "total_reviews": - return 312 - raise NotImplementedError(f"Metric {metric} not supported") + code = "itunes" + + def __init__(self): + super().__init__() + source_data = SourceService().get_source_data(code=self.code) + self.name = source_data["name"] + self.type = source_data["type"] + self.url = source_data["url"] + self.supported_metrics = source_data["supported_metrics"] + + def supports_metric(self, metric: str): + return metric in self.supported_metrics + + def lookup_app(self, appstore_id: str): + response = requests.get(f"{self.url}/lookup?id={appstore_id}") + if not response.ok: + return None + + data = response.json() + results = data.get("results", []) + return results[0] if results else None + + def fetch(self, app_id: int, metrics: list[str]): + app = AppService().get_app(app_id=app_id) + if not app: + return {} + + result = self.lookup_app(app.appstore_id) + if not result: + return {} + + metric_map = { + MetricCodes.AVERAGE_RATING: str(result.get("averageUserRating") or ""), + MetricCodes.TOTAL_REVIEWS: str(result.get("userRatingCount") or ""), + } + + return { + metric: metric_map.get(metric, "") for metric in metrics if self.supports_metric(metric) + } diff --git a/source/adapters/news.py b/source/adapters/news.py new file mode 100644 index 0000000..1bdea50 --- /dev/null +++ b/source/adapters/news.py @@ -0,0 +1,58 @@ +import os +from datetime import datetime, timedelta + +import requests + +from app.services import AppService +from metric.constants import MetricCodes +from source.adapters.base import SourceAdapter +from source.services import SourceService + + +class NewsAPIAdapter(SourceAdapter): + code = "news" + + def __init__(self, api_key=None): + super().__init__() + source_data = SourceService().get_source_data(code=self.code) + self.name = source_data["name"] + self.type = source_data["type"] + self.url = source_data["url"] + self.supported_metrics = source_data["supported_metrics"] + self.api_key = api_key or os.environ.get("NEWSAPI_KEY") + + def supports_metric(self, metric: str) -> bool: + return metric in self.supported_metrics + + def fetch(self, app_id: int, metrics: list[str]): + app = AppService().get_app(app_id=app_id) + if not app: + return {} + + if not self.api_key: + return {} + + yesterday = (datetime.today() - timedelta(days=1)).strftime("%Y-%m-%d") + params = { + "q": app.code, + "from": yesterday, + "to": yesterday, + "sortBy": "publishedAt", + "language": "en", + "apiKey": self.api_key, + } + + response = requests.get(f"{self.url}/everything", params=params) + + if not response.ok: + return {} + + data = response.json() + + metric_map = { + MetricCodes.DAILY_NEWS_BLOG_MENTIONS: str(data.get("totalResults", 0)), + } + + return { + metric: metric_map.get(metric, "") for metric in metrics if self.supports_metric(metric) + } diff --git a/source/adapters/reddit.py b/source/adapters/reddit.py new file mode 100644 index 0000000..52613dd --- /dev/null +++ b/source/adapters/reddit.py @@ -0,0 +1,72 @@ +import os + +import praw +from rest_framework.exceptions import APIException + +from app.services import AppService +from metric.constants import MetricCodes +from source.adapters.base import SourceAdapter +from source.services import SourceService + + +class RedditAPIAdapter(SourceAdapter): + code = "reddit" + + def __init__(self): + super().__init__() + source_data = SourceService().get_source_data(code=self.code) + self.name = source_data["name"] + self.type = source_data["type"] + self.url = source_data["url"] + self.supported_metrics = source_data["supported_metrics"] + try: + self.reddit = praw.Reddit( + client_id=os.getenv("REDDIT_CLIENT_ID"), + client_secret=os.getenv("REDDIT_CLIENT_SECRET"), + user_agent=os.getenv("REDDIT_USER_AGENT"), + username=os.getenv("REDDIT_USERNAME"), + password=os.getenv("REDDIT_PASSWORD"), + ) + except KeyError as e: + raise APIException(f"Reddit adapter initialization failed: missing env var {str(e)}.") + except praw.exceptions.MissingRequiredAttributeException as e: + raise APIException(f"Reddit adapter configuration error: {str(e)}") + + def supports_metric(self, metric: str) -> bool: + return metric in self.supported_metrics + + def fetch(self, app_id: int, metrics: list[str]): + app = AppService().get_app(app_id=app_id) + if not app: + return {} + + query = app.name.lower() + total = 0 + after = None + try: + while True: + results = self.reddit.subreddit("all").search( + query, + time_filter="day", + sort="new", + limit=100, + params={"after": after} if after else {}, + ) + batch = list(results) + total += len(batch) + + if len(batch) < 100: + break + + after = batch[-1].fullname # exemple: 't3_abcdef' + except Exception as e: + print(f"[RedditAPIAdapter] Error fetching search data: {e}") + return {} + + metric_map = { + MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS: str(total), + } + + return { + metric: metric_map.get(metric, "") for metric in metrics if self.supports_metric(metric) + } diff --git a/source/apps.py b/source/apps.py index 06d886d..4bbfa3d 100644 --- a/source/apps.py +++ b/source/apps.py @@ -4,3 +4,7 @@ class SourceConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "source" + + # def ready(self): + # from source.services.source_registration_service import SourceRegistrationService + # SourceRegistrationService().load_and_register() diff --git a/source/constants/__init__.py b/source/constants/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/source/constants/source_type.py b/source/constants/source_type.py new file mode 100644 index 0000000..acfa338 --- /dev/null +++ b/source/constants/source_type.py @@ -0,0 +1,7 @@ +from django.db import models + + +class SourceType(models.TextChoices): + API = "api", "API" + SCRAPER = "scraper", "Scraper" + EXTERNAL_TOOL = "external_tool", "External Tool" diff --git a/source/migrations/0002_source_code.py b/source/migrations/0002_source_code.py new file mode 100644 index 0000000..e4cdfe1 --- /dev/null +++ b/source/migrations/0002_source_code.py @@ -0,0 +1,19 @@ +# Generated by Django 5.1.7 on 2025-04-05 14:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("source", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="source", + name="code", + field=models.CharField(default="default_code", max_length=100, unique=True), + preserve_default=False, + ), + ] diff --git a/source/migrations/0003_source_metrics.py b/source/migrations/0003_source_metrics.py new file mode 100644 index 0000000..f261457 --- /dev/null +++ b/source/migrations/0003_source_metrics.py @@ -0,0 +1,19 @@ +# Generated by Django 5.1.7 on 2025-04-16 16:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("metric", "0003_alter_metric_code"), + ("source", "0002_source_code"), + ] + + operations = [ + migrations.AddField( + model_name="source", + name="metrics", + field=models.ManyToManyField(related_name="sources", to="metric.metric"), + ), + ] diff --git a/source/models.py b/source/models.py index 1a36777..1c3a1b5 100644 --- a/source/models.py +++ b/source/models.py @@ -1,16 +1,14 @@ from django.db import models - -class SourceType(models.TextChoices): - API = "api", "API" - SCRAPER = "scraper", "Scraper" - EXTERNAL_TOOL = "external_tool", "External Tool" +from source.constants.source_type import SourceType class Source(models.Model): + code = models.CharField(max_length=100, unique=True) name = models.CharField(max_length=100) type = models.CharField(max_length=20, choices=SourceType.choices) url = models.URLField(null=True, blank=True) + metrics = models.ManyToManyField("metric.Metric", related_name="sources") def __str__(self): return self.name diff --git a/source/repositories.py b/source/repositories.py new file mode 100644 index 0000000..4ea641b --- /dev/null +++ b/source/repositories.py @@ -0,0 +1,43 @@ +from .models import Source + + +class SourceRepository: + + @staticmethod + def get_all(): + return Source.objects.all() + + @staticmethod + def get_by_id(app_id): + return Source.objects.get(id=app_id) + + @staticmethod + def create(data): + return Source.objects.create(**data) + + @staticmethod + def update(instance, data): + for attr, value in data.items(): + setattr(instance, attr, value) + instance.save() + return instance + + @staticmethod + def delete(instance): + instance.delete() + + @staticmethod + def add_metrics(instance, metrics_ids): + instance.metrics.add(*metrics_ids) + instance.save() + return instance + + @staticmethod + def remove_metrics(instance, metrics_ids): + instance.metrics.remove(*metrics_ids) + instance.save() + return instance + + @staticmethod + def get_by_code(code: str): + return Source.objects.prefetch_related("metrics").get(code=code) diff --git a/source/serializers.py b/source/serializers.py new file mode 100644 index 0000000..10814a2 --- /dev/null +++ b/source/serializers.py @@ -0,0 +1,34 @@ +from rest_framework import serializers + +from .constants.source_type import SourceType +from .models import Source + + +class SourceSerializer(serializers.ModelSerializer): + type = serializers.ChoiceField( + choices=SourceType.choices, + help_text=( + "Type of the source. Available options:\n" + "- 'api': Data source accessible through a public API.\n" + "- 'scraper': Data source obtained via web scraping.\n" + "- 'external_tool': Data provided by an external tool." + ), + ) + + class Meta: + model = Source + fields = [ + "id", + "code", + "name", + "type", + "url", + "metrics", + ] + read_only_fields = ["metrics"] + + +class LinkSourceSerializer(serializers.Serializer): + sources = serializers.ListField( + child=serializers.IntegerField(), help_text="List of source IDs to link to the metric." + ) diff --git a/source/services.py b/source/services.py new file mode 100644 index 0000000..daad8cb --- /dev/null +++ b/source/services.py @@ -0,0 +1,60 @@ +from django.core.exceptions import ObjectDoesNotExist +from rest_framework.exceptions import NotFound + +from source.adapters.base import SourceAdapter + +from .repositories import SourceRepository + + +class SourceService: + def __init__(self): + self.repo = SourceRepository() + + def list_sources(self): + return self.repo.get_all() + + def get_source(self, source_id): + try: + return self.repo.get_by_id(source_id) + except ObjectDoesNotExist: + raise NotFound(f"The source with ID '{source_id}' is not registered.") + + def create_source(self, validated_data): + return self.repo.create(validated_data) + + def update_source(self, instance, validated_data): + return self.repo.update(instance, validated_data) + + def delete_source(self, instance): + return self.repo.delete(instance) + + def add_metrics(self, instance, metrics_ids): + return self.repo.add_metrics(instance, metrics_ids) + + def remove_metrics(self, instance, metrics_ids): + return self.repo.remove_metrics(instance, metrics_ids) + + def get_source_data(self, code: str) -> dict: + try: + source = self.repo.get_by_code(code) + except ObjectDoesNotExist: + raise NotFound(f"The source with code '{code}' is not registered.") + + return { + "code": source.code, + "name": source.name, + "type": source.type, + "url": source.url, + "supported_metrics": list(source.metrics.values_list("code", flat=True)), + } + + @staticmethod + def load_sources(): + # isort: off + import source.adapters.itunes as _ # noqa + import source.adapters.news as _ # noqa + import source.adapters.reddit as _ # noqa + import source.adapters.google_play_scraper as _ # noqa + + # isort: on + return [cls() for cls in SourceAdapter.__subclasses__()] diff --git a/source/tests.py b/source/tests.py index e69de29..aa2a8f0 100644 --- a/source/tests.py +++ b/source/tests.py @@ -0,0 +1,19 @@ +import pytest + +from source.constants.source_type import SourceType +from source.models import Source + + +@pytest.mark.django_db +def test_create_source(): + s = Source.objects.create( + code="test_source", + name="Test Source", + type=SourceType.API, + url="https://example.com", + ) + assert s.pk is not None + assert s.code == "test_source" + assert s.name == "Test Source" + assert s.type == SourceType.API + assert s.url == "https://example.com" diff --git a/source/urls.py b/source/urls.py new file mode 100644 index 0000000..a969b6a --- /dev/null +++ b/source/urls.py @@ -0,0 +1,11 @@ +from django.urls import include, path +from rest_framework.routers import DefaultRouter + +from .views import SourceViewSet + +router = DefaultRouter() +router.register(r"sources", SourceViewSet, basename="source") + +urlpatterns = [ + path("", include(router.urls)), +] diff --git a/source/views.py b/source/views.py index e69de29..5672410 100644 --- a/source/views.py +++ b/source/views.py @@ -0,0 +1,112 @@ +from drf_spectacular.utils import OpenApiParameter, extend_schema +from rest_framework import status, viewsets +from rest_framework.decorators import action +from rest_framework.response import Response + +from metric.serializers import LinkMetricsSerializer + +from .serializers import SourceSerializer +from .services import SourceService + + +class SourceViewSet(viewsets.ViewSet): + service = SourceService() + + @extend_schema( + responses=SourceSerializer(many=True), + tags=["Sources"], + ) + def list(self, request): + sources = self.service.list_sources() + serializer = SourceSerializer(sources, many=True) + return Response(serializer.data) + + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses=SourceSerializer, + tags=["Sources"], + ) + def retrieve(self, request, pk=None): + source = self.service.get_source(pk) + serializer = SourceSerializer(source) + return Response(serializer.data) + + @extend_schema( + request=SourceSerializer, + responses=SourceSerializer, + description=( + "Creates a new data source.\n\n" + "The 'metrics' field is read-only and cannot be set directly on creation." + ), + tags=["Sources"], + ) + def create(self, request): + serializer = SourceSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + source = self.service.create_source(serializer.validated_data) + return Response(SourceSerializer(source).data, status=status.HTTP_201_CREATED) + + @extend_schema( + request=SourceSerializer, + responses=SourceSerializer, + tags=["Sources"], + ) + def update(self, request, pk=None): + source = self.service.get_source(pk) + serializer = SourceSerializer(source, data=request.data) + serializer.is_valid(raise_exception=True) + updated_source = self.service.update_source(source, serializer.validated_data) + return Response(SourceSerializer(updated_source).data) + + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses={204: None}, + tags=["Sources"], + ) + def destroy(self, request, pk=None): + source = self.service.get_source(pk) + source = self.service.delete_source(source) + return Response(status=status.HTTP_204_NO_CONTENT) + + @extend_schema( + request=LinkMetricsSerializer, + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses=SourceSerializer, + description="Adds a list of metric IDs to an existing " + "source without removing existing ones.", + tags=["Sources"], + ) + @action(detail=True, methods=["post"], url_path="metrics") + def add_metrics(self, request, pk=None): + metrics_ids = request.data.get("metrics", []) + + if not isinstance(metrics_ids, list): + return Response( + {"error": "metrics must be a list of IDs"}, status=status.HTTP_400_BAD_REQUEST + ) + + source = self.service.get_source(pk) + source = self.service.add_metrics(source, metrics_ids) + + return Response(SourceSerializer(source).data, status=status.HTTP_200_OK) + + @extend_schema( + request=LinkMetricsSerializer, + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses=SourceSerializer, + description="Removes a list of metric IDs from an existing source.", + tags=["Sources"], + ) + @action(detail=True, methods=["delete"], url_path="metrics") + def remove_metrics(self, request, pk=None): + metrics_ids = request.data.get("metrics", []) + + if not isinstance(metrics_ids, list): + return Response( + {"error": "metrics must be a list of IDs"}, status=status.HTTP_400_BAD_REQUEST + ) + + source = self.service.get_source(pk) + source = self.service.remove_metrics(source, metrics_ids) + + return Response(SourceSerializer(source).data, status=status.HTTP_200_OK) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/adapters/__init__.py b/tests/adapters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/adapters/test_all_adapters.py b/tests/adapters/test_all_adapters.py new file mode 100644 index 0000000..7015a2f --- /dev/null +++ b/tests/adapters/test_all_adapters.py @@ -0,0 +1,27 @@ +import pytest + +from source.adapters.base import SourceAdapter + +# isort: off +import source.adapters.itunes as _ # noqa +import source.adapters.news as _ # noqa +import source.adapters.reddit as _ # noqa +import source.adapters.google_play_scraper as _ # noqa + + +# isort: on + + +@pytest.mark.django_db +@pytest.mark.parametrize("adapter_class", SourceAdapter.__subclasses__()) +def test_fetch_returns_dict_of_strings(adapter_class, dummy_app): + + adapter = adapter_class() + if not adapter.supported_metrics: + pytest.skip(f"No supported metrics for {adapter.code}") + result = adapter.fetch(app_id=dummy_app.id, metrics=adapter.supported_metrics) + assert isinstance(result, dict) + assert result is not None, f"{adapter.code}.fetch() returned None" + for k, v in result.items(): + assert isinstance(k, str), f"Key {k} in {adapter.code} is not a string" + assert isinstance(v, str), f"Value for {k} in {adapter.code} is not a string" diff --git a/tests/adapters/test_google_play_scraper.py b/tests/adapters/test_google_play_scraper.py new file mode 100644 index 0000000..45027d0 --- /dev/null +++ b/tests/adapters/test_google_play_scraper.py @@ -0,0 +1,127 @@ +from datetime import datetime +from unittest.mock import patch + +import pytest +from rest_framework.exceptions import ParseError, ValidationError + +from metric.constants import MetricCodes +from source.adapters.google_play_scraper import GooglePlayScraperAdapter + + +@patch("app.models.App.objects.get") +@patch("source.adapters.google_play_scraper.gp_app") +def test_google_play_fetch(mock_gp_app, mock_app_get, dummy_app): + """Testa que GooglePlayScraperAdapter.fetch retorna valors esperats amb mocks.""" + + mock_app_get.return_value.playstore_id = "com.whatsapp" + mock_gp_app.return_value = { + "score": 4.6, + "reviews": 50000, + "realInstalls": 1000000, + "lastUpdatedOn": "2024-12-10", + } + + adapter = GooglePlayScraperAdapter() + + assert adapter.supports_metric(MetricCodes.AVERAGE_RATING) + + result = adapter.fetch( + dummy_app.id, + [ + MetricCodes.AVERAGE_RATING, + MetricCodes.TOTAL_REVIEWS, + MetricCodes.TOTAL_DOWNLOADS, + MetricCodes.LAST_UPDATE_DATE, + ], + ) + + assert result == { + MetricCodes.AVERAGE_RATING: "4.6", + MetricCodes.TOTAL_REVIEWS: "50000", + MetricCodes.TOTAL_DOWNLOADS: "1000000", + MetricCodes.LAST_UPDATE_DATE: "2024-12-10", + } + + unsupported = adapter.fetch(dummy_app.id, ["unsupported_metric"]) + assert unsupported == {} + + +@patch("app.models.App.objects.get") +@patch("source.adapters.google_play_scraper.reviews") +def test_google_play_fetch_reviews(mock_reviews, mock_app_get, dummy_app): + """Testa que GooglePlayScraperAdapter.fetch_reviews retorna reviews dins del rang.""" + + mock_app_get.return_value.playstore_id = "com.whatsapp" + + # Simulem que es retornen dues tandes de reviews, amb una data dins el rang + mock_reviews.side_effect = [ + ( + [ + {"userName": "Anna", "content": "Molt bona!", "at": datetime(2024, 5, 3)}, + {"userName": "Pau", "content": "No funciona bé", "at": datetime(2024, 5, 2)}, + ], + "token123", + ), + ( + [ + {"userName": "Joan", "content": "Ok", "at": datetime(2024, 5, 1)}, + ], + None, + ), + ] + + adapter = GooglePlayScraperAdapter() + result = adapter.fetch_reviews(dummy_app.id, date_from="2024-05-01", date_to="2024-05-03") + + # Ha de contenir les 3 reviews simulades + assert isinstance(result, list) + assert len(result) == 3 + assert all("content" in r for r in result) + assert all("userName" in r for r in result) + + +@patch("app.models.App.objects.get") +@patch("source.adapters.google_play_scraper.reviews") +def test_fetch_reviews_stops_on_review_before_date_from(mock_reviews, mock_app_get, dummy_app): + """Ha de tallar si una review és anterior a date_from.""" + + mock_app_get.return_value.playstore_id = "com.whatsapp" + mock_reviews.return_value = ( + [{"content": "Antiga", "at": datetime(2024, 4, 30)}], # fora del rang + None, + ) + + adapter = GooglePlayScraperAdapter() + result = adapter.fetch_reviews(dummy_app.id, date_from="2024-05-01", date_to="2024-05-03") + + assert result == [] # Ha de sortir immediatament + + +@patch("app.models.App.objects.get") +@patch("source.adapters.google_play_scraper.reviews") +def test_fetch_reviews_returns_empty_list_if_no_reviews(mock_reviews, mock_app_get, dummy_app): + mock_app_get.return_value.playstore_id = "com.whatsapp" + mock_reviews.return_value = ([], None) + + adapter = GooglePlayScraperAdapter() + result = adapter.fetch_reviews(dummy_app.id) + + assert result == [] + + +@patch("app.models.App.objects.get") +def test_fetch_reviews_raises_parse_error_on_invalid_date(mock_app_get, dummy_app): + mock_app_get.return_value.playstore_id = "com.whatsapp" + adapter = GooglePlayScraperAdapter() + + with pytest.raises(ParseError): + adapter.fetch_reviews(dummy_app.id, date_from="invalid-date") + + +@patch("app.models.App.objects.get") +def test_fetch_reviews_raises_validation_error_if_date_range_invalid(mock_app_get, dummy_app): + mock_app_get.return_value.playstore_id = "com.whatsapp" + adapter = GooglePlayScraperAdapter() + + with pytest.raises(ValidationError): + adapter.fetch_reviews(dummy_app.id, date_from="2024-05-05", date_to="2024-05-01") diff --git a/tests/adapters/test_itunes_adapter.py b/tests/adapters/test_itunes_adapter.py new file mode 100644 index 0000000..ba448ef --- /dev/null +++ b/tests/adapters/test_itunes_adapter.py @@ -0,0 +1,26 @@ +from unittest.mock import patch + +from metric.constants import MetricCodes +from source.adapters.itunes import ItunesSearchAPIAdapter + + +@patch("app.models.App.objects.get") +@patch("source.adapters.itunes.requests.get") +def test_itunes_fetch(mock_requests_get, mock_app_get, dummy_app): + """Testa que ItunesSearchAPIAdapter.fetch retorna valors esperats amb mocks.""" + mock_app_get.return_value.appstore_id = "123" + mock_requests_get.return_value.ok = True + mock_requests_get.return_value.json.return_value = { + "resultCount": 1, + "results": [{"averageUserRating": 4.5, "userRatingCount": 1000}], + } + + adapter = ItunesSearchAPIAdapter() + + assert adapter.supports_metric(MetricCodes.AVERAGE_RATING) + + result = adapter.fetch(dummy_app.id, [MetricCodes.AVERAGE_RATING, MetricCodes.TOTAL_REVIEWS]) + assert result == {MetricCodes.AVERAGE_RATING: "4.5", MetricCodes.TOTAL_REVIEWS: "1000"} + + unsupported = adapter.fetch(dummy_app.id, ["unsupported_metric"]) + assert unsupported == {} diff --git a/tests/adapters/test_news.py b/tests/adapters/test_news.py new file mode 100644 index 0000000..39cb929 --- /dev/null +++ b/tests/adapters/test_news.py @@ -0,0 +1,31 @@ +from unittest.mock import patch + +from metric.constants import MetricCodes +from source.adapters.news import NewsAPIAdapter + + +@patch("app.models.App.objects.get") +@patch("source.adapters.news.requests.get") +def test_news_adapter_fetch(mock_requests_get, mock_app_get, dummy_app): + """Testa que NewsAPIAdapter.fetch retorna valors esperats amb mocks.""" + + # Simulem app amb codi "whatsapp" + mock_app_get.return_value.code = "whatsapp" + + # Simulem resposta de l'API de NewsAPI + mock_requests_get.return_value.ok = True + mock_requests_get.return_value.json.return_value = { + "status": "ok", + "totalResults": 42, + "articles": [{}] * 42, + } + + adapter = NewsAPIAdapter("fake-api-key") + + result = adapter.fetch(dummy_app.id, [MetricCodes.DAILY_NEWS_BLOG_MENTIONS]) + + assert isinstance(result, dict) + assert result.get(MetricCodes.DAILY_NEWS_BLOG_MENTIONS) == "42" + + unsupported = adapter.fetch(dummy_app.id, ["unsupported_metric"]) + assert unsupported == {} diff --git a/tests/adapters/test_reddit.py b/tests/adapters/test_reddit.py new file mode 100644 index 0000000..625b778 --- /dev/null +++ b/tests/adapters/test_reddit.py @@ -0,0 +1,31 @@ +from unittest.mock import MagicMock, patch + +from metric.constants import MetricCodes +from source.adapters.reddit import RedditAPIAdapter + + +@patch("app.models.App.objects.get") +@patch("praw.Reddit") +def test_reddit_adapter_fetch(mock_reddit_class, mock_app_get, dummy_app): + """Testa que RedditAPIAdapter.fetch retorna valors esperats amb mocks.""" + + # Simulem app + mock_app_get.return_value.name = "WhatsApp" + + # Simulem la instància Reddit i el seu comportament + mock_reddit = MagicMock() + mock_results = [{}] * 23 # 23 mencions + mock_reddit.subreddit.return_value.search.return_value = mock_results + mock_reddit_class.return_value = mock_reddit + + metric_code = MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS + + # ✅ Instanciar adapter després de tenir la BD a punt + adapter = RedditAPIAdapter() + + result = adapter.fetch(dummy_app.id, [metric_code]) + assert isinstance(result, dict) + assert result.get(metric_code) == "23" + + unsupported = adapter.fetch(dummy_app.id, ["unsupported_metric"]) + assert unsupported == {} diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9b107a7 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,43 @@ +import pytest + +from app.models import App +from metric.constants import MetricCodes +from metric.models import Metric +from source.models import Source + + +@pytest.fixture +def dummy_app(db): + return App.objects.create( + code="dummy", name="Dummy App", appstore_id="310633997", playstore_id="com.whatsapp" + ) + + +@pytest.fixture(autouse=True) +def create_default_sources_and_metrics(db): + """Crea sources i mètriques per defecte disponibles a tots els tests.""" + + itunes = Source.objects.create( + code="itunes", name="App Store", type="api", url="https://itunes.apple.com" + ) + google = Source.objects.create( + code="google_play", name="Google Play", type="scraper", url="https://play.google.com" + ) + news = Source.objects.create(code="news", name="NewsAPI", type="api", url="https://newsapi.org") + reddit = Source.objects.create( + code="reddit", name="Reddit", type="api", url="https://reddit.com" + ) + + for metric_code in [ + MetricCodes.AVERAGE_RATING, + MetricCodes.TOTAL_REVIEWS, + MetricCodes.TOTAL_DOWNLOADS, + MetricCodes.LAST_UPDATE_DATE, + MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS, + MetricCodes.DAILY_NEWS_BLOG_MENTIONS, + ]: + metric = Metric.objects.create(code=metric_code, name=metric_code.replace("_", " ").title()) + itunes.metrics.add(metric) + google.metrics.add(metric) + news.metrics.add(metric) + reddit.metrics.add(metric) diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py new file mode 100644 index 0000000..9ed79df --- /dev/null +++ b/tests/integration/test_end_to_end.py @@ -0,0 +1,21 @@ +import pytest + +from source.services import SourceService + + +@pytest.mark.django_db +def test_full_flow_with_real_adapter(dummy_app): + # Adapter real (ja ha de formar part del projecte i registrar-se automàticament) + adapters = SourceService().load_sources() + itunes_adapter = next((a for a in adapters if a.code == "itunes"), None) + + assert itunes_adapter is not None, "iTunes adapter not found in loaded sources" + + values = itunes_adapter.fetch(dummy_app.id, ["average_rating"]) + + # Comprovar que el valor retornat és un diccionari i conté la clau "average_rating" + assert isinstance(values, dict), "Expected values to be a dictionary" + assert "average_rating" in values, "'average_rating' key not found in values" + + # Comprovar que el valor no és None + assert values["average_rating"] is not None, "'average_rating' value is None" diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_source_registration_service.py b/tests/unit/test_source_registration_service.py new file mode 100644 index 0000000..ddaa568 --- /dev/null +++ b/tests/unit/test_source_registration_service.py @@ -0,0 +1,24 @@ +from source.services import SourceService + + +class FakeRepository: + def __init__(self): + self.registered = [] + + def get_or_create(self, code, defaults): + self.registered.append((code, defaults)) + return None + + +""" +def test_source_registration_service_registers_adapters(): + repo = FakeRepository() + service = SourceService(repository=repo) + adapters = service.load_and_register() + assert len(repo.registered) == len(adapters) +""" + + +def test_load_sources_works(): + adapters = SourceService.load_sources() + assert len(adapters) > 0