From 8a9abb693a5d351b838ea0445192e01c1aead17e Mon Sep 17 00:00:00 2001 From: Anyer Date: Sat, 5 Apr 2025 17:31:36 +0200 Subject: [PATCH 01/42] feat(data-source): Implementar la arquitectura de software completa Refs #99 --- app/migrations/0002_app_code.py | 19 +++++++ app/models.py | 1 + metric/constants/__init__.py | 3 ++ metric/constants/value_types.py | 9 ++++ metric/migrations/0002_metric_code.py | 19 +++++++ metric/migrations/0003_alter_metric_code.py | 18 +++++++ metric/models.py | 10 +--- metric/registry/metric_registry.py | 35 ++++++++++++ metric/registry/source_registry.py | 14 ----- metric/strategies/base.py | 2 +- .../{average_rating.py => generic.py} | 14 ++--- requirements.txt | Bin 204 -> 940 bytes source/adapters/base.py | 10 +++- source/adapters/itunes.py | 33 +++++++++--- source/constants/__init__.py | 0 source/constants/source_type.py | 7 +++ source/migrations/0002_source_code.py | 19 +++++++ source/models.py | 7 +-- source/registry/__init__.py | 0 source/registry/source_registry.py | 50 ++++++++++++++++++ 20 files changed, 229 insertions(+), 41 deletions(-) create mode 100644 app/migrations/0002_app_code.py create mode 100644 metric/constants/__init__.py create mode 100644 metric/constants/value_types.py create mode 100644 metric/migrations/0002_metric_code.py create mode 100644 metric/migrations/0003_alter_metric_code.py create mode 100644 metric/registry/metric_registry.py delete mode 100644 metric/registry/source_registry.py rename metric/strategies/{average_rating.py => generic.py} (57%) create mode 100644 source/constants/__init__.py create mode 100644 source/constants/source_type.py create mode 100644 source/migrations/0002_source_code.py create mode 100644 source/registry/__init__.py create mode 100644 source/registry/source_registry.py diff --git a/app/migrations/0002_app_code.py b/app/migrations/0002_app_code.py new file mode 100644 index 0000000..f9cc194 --- /dev/null +++ b/app/migrations/0002_app_code.py @@ -0,0 +1,19 @@ +# Generated by Django 5.1.7 on 2025-04-05 14:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("app", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="app", + name="code", + field=models.CharField(default="default_code", max_length=100, unique=True), + preserve_default=False, + ), + ] diff --git a/app/models.py b/app/models.py index 4b8bd84..1e835ab 100644 --- a/app/models.py +++ b/app/models.py @@ -2,6 +2,7 @@ class App(models.Model): + code = models.CharField(max_length=100, unique=True) name = models.CharField(max_length=200) description = models.TextField() appstore_id = models.CharField(max_length=100, null=True, unique=True) diff --git a/metric/constants/__init__.py b/metric/constants/__init__.py new file mode 100644 index 0000000..e4114a4 --- /dev/null +++ b/metric/constants/__init__.py @@ -0,0 +1,3 @@ +class MetricCodes: + AVERAGE_RATING = "average_rating" + TOTAL_REVIEWS = "total_reviews" diff --git a/metric/constants/value_types.py b/metric/constants/value_types.py new file mode 100644 index 0000000..752b940 --- /dev/null +++ b/metric/constants/value_types.py @@ -0,0 +1,9 @@ +from django.db import models + + +class MetricValueType(models.TextChoices): + STRING = "string", "String" + INTEGER = "integer", "Integer" + FLOAT = "float", "Float" + DATE = "date", "Date" + BOOLEAN = "boolean", "Boolean" diff --git a/metric/migrations/0002_metric_code.py b/metric/migrations/0002_metric_code.py new file mode 100644 index 0000000..a77834c --- /dev/null +++ b/metric/migrations/0002_metric_code.py @@ -0,0 +1,19 @@ +# Generated by Django 5.1.7 on 2025-04-04 18:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("metric", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="metric", + name="code", + field=models.CharField(default="default_code", max_length=100), + preserve_default=False, + ), + ] diff --git a/metric/migrations/0003_alter_metric_code.py b/metric/migrations/0003_alter_metric_code.py new file mode 100644 index 0000000..154fa17 --- /dev/null +++ b/metric/migrations/0003_alter_metric_code.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.7 on 2025-04-05 14:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("metric", "0002_metric_code"), + ] + + operations = [ + migrations.AlterField( + model_name="metric", + name="code", + field=models.CharField(max_length=100, unique=True), + ), + ] diff --git a/metric/models.py b/metric/models.py index 72ad355..1f9269b 100644 --- a/metric/models.py +++ b/metric/models.py @@ -1,18 +1,12 @@ from django.db import models from app.models import App +from metric.constants.value_types import MetricValueType from source.models import Source -class MetricValueType(models.TextChoices): - STRING = "string", "String" - INTEGER = "integer", "Integer" - FLOAT = "float", "Float" - DATE = "date", "Date" - BOOLEAN = "boolean", "Boolean" - - class Metric(models.Model): + code = models.CharField(max_length=100, unique=True) name = models.CharField(max_length=100) description = models.TextField() value_type = models.CharField(max_length=10, choices=MetricValueType.choices) diff --git a/metric/registry/metric_registry.py b/metric/registry/metric_registry.py new file mode 100644 index 0000000..f986e49 --- /dev/null +++ b/metric/registry/metric_registry.py @@ -0,0 +1,35 @@ +from metric.constants import MetricCodes +from metric.constants.value_types import MetricValueType +from metric.models import Metric + +METRICS = [ + { + "code": MetricCodes.AVERAGE_RATING, + "name": "Average Rating", + "description": "Valoració mitjana de l'app", + "value_type": MetricValueType.FLOAT, + }, + { + "code": MetricCodes.TOTAL_REVIEWS, + "name": "Total Reviews", + "description": "Nombre total de ressenyes", + "value_type": MetricValueType.INTEGER, + }, + # Afegir aquí les noves +] + + +def register_metrics(): + for m in METRICS: + metric, created = Metric.objects.get_or_create( + code=m["code"], + defaults={ + "name": m["name"], + "description": m["description"], + "value_type": m["value_type"].value, + }, + ) + if created: + print(f"✅ Mètrica registrada: {m['code']}") + else: + print(f"ℹ️ Ja existia: {m['code']}") diff --git a/metric/registry/source_registry.py b/metric/registry/source_registry.py deleted file mode 100644 index d119020..0000000 --- a/metric/registry/source_registry.py +++ /dev/null @@ -1,14 +0,0 @@ -from typing import List - -from source.adapters.base import SourceAdapter - - -class SourceRegistry: - def __init__(self, adapters: List[SourceAdapter]): - self.adapters = adapters - - def get_sources(self) -> List[SourceAdapter]: - return self.adapters - - def get_for_metric(self, metric: str) -> List[SourceAdapter]: - return [adapter for adapter in self.adapters if adapter.supports_metric(metric)] diff --git a/metric/strategies/base.py b/metric/strategies/base.py index 290d1d1..6921001 100644 --- a/metric/strategies/base.py +++ b/metric/strategies/base.py @@ -2,7 +2,7 @@ from typing import List from metric.models import MetricValue -from metric.registry.source_registry import SourceRegistry +from source.registry.source_registry import SourceRegistry class MetricStrategy(ABC): diff --git a/metric/strategies/average_rating.py b/metric/strategies/generic.py similarity index 57% rename from metric/strategies/average_rating.py rename to metric/strategies/generic.py index 88a0fac..0ebe5f0 100644 --- a/metric/strategies/average_rating.py +++ b/metric/strategies/generic.py @@ -1,19 +1,21 @@ from datetime import datetime from metric.models import MetricValue -from metric.registry.source_registry import SourceRegistry -from metric.strategies.base import MetricStrategy +from source.registry.source_registry import SourceRegistry -class AverageRatingMetric(MetricStrategy): +class GenericMetricStrategy: + def __init__(self, metric_code: str): + self.metric_code = metric_code + def compute_all(self, app_id: str, registry: SourceRegistry) -> list[MetricValue]: results = [] - for adapter in registry.get_for_metric("average_rating"): - value = adapter.fetch(app_id, "average_rating") + for adapter in registry.get_for_metric(self.metric_code): + value = adapter.fetch(app_id, self.metric_code) results.append( MetricValue( app_id=app_id, - metric_code="average_rating", + metric_code=self.metric_code, source_name=adapter.__class__.__name__, value=value, retrieved_at=datetime.now(), diff --git a/requirements.txt b/requirements.txt index fa611434bda1a8d13070373acd911f53c9aa46a6..7dab18512b160fe3fe698398a48b34ad8d6a3db3 100644 GIT binary patch literal 940 zcmY*XO;3YR5S+7#KZT}%R(tSZ;!zW?o(!e*OA1(MWBu{!%K#32-GTa(6!PL-_T?Y*8U-40QMr5qY14~JKjgwV~ z|AtRw*pUA~6%~0BG-&y6&u=vuzl}etQkSbL4UMW?tyD{KNru{33gc1tG|Ze?#gfcD zr)k2%Gzpikx#3UPaehO^t)bELUFdGvTM>6Z>YZPDl|xNb3ZgzSr;?t&gl3gRpABmX6Vha5-_l zj`QW9u=`LNzF%!2vm{h8U_ z+Xk0DFW!qBYNlKz)c=1Ma^Nm?39qbu!y{hsZ1ZAXu3pnVapYUK;EJR>r#rpH(6geZ zyLh4TmUIzbG4Mv0=Fqn_-oH;d(pAl$E-g0|&K~`oERTUYbn)XZ#G@H{`@a4FmArI{^SXSOtUt diff --git a/source/adapters/base.py b/source/adapters/base.py index e1611a9..12bd8e5 100644 --- a/source/adapters/base.py +++ b/source/adapters/base.py @@ -1,7 +1,15 @@ from typing import Protocol +from source.constants.source_type import SourceType + class SourceAdapter(Protocol): + code: str + name: str + type: SourceType + url: str | None + supported_metrics: list[str] + def supports_metric(self, metric: str) -> bool: ... - def fetch(self, app_name: str, metric: str) -> str: ... + def fetch(self, app_id: str, metric: str) -> str: ... diff --git a/source/adapters/itunes.py b/source/adapters/itunes.py index d4675c9..407436b 100644 --- a/source/adapters/itunes.py +++ b/source/adapters/itunes.py @@ -1,13 +1,34 @@ +import requests + +from app.models import App +from metric.constants import MetricCodes from source.adapters.base import SourceAdapter +from source.constants.source_type import SourceType class ItunesSearchAPIAdapter(SourceAdapter): + code = "itunes" + name = "App Store" + type = SourceType.API + url = "https://itunes.apple.com/" + supported_metrics = [MetricCodes.AVERAGE_RATING, MetricCodes.TOTAL_REVIEWS] + def supports_metric(self, metric: str) -> bool: - return metric in ["average_rating", "total_reviews"] + return metric in self.supported_metrics + + def fetch(self, app_name: str, metric: str): + app = App.objects.get(name=app_name) + response = requests.get(f"{self.url}lookup?id={app.appstore_id}") + data = response.json() + + if data.get("resultCount", 0) == 0 or "results" not in data: + return None # Protegeix si la resposta no té resultats + + result = data["results"][0] + + if metric == MetricCodes.AVERAGE_RATING: + return result.get("averageUserRating", None) + elif metric == MetricCodes.TOTAL_REVIEWS: + return result.get("userRatingCount", None) - def fetch(self, app_id: str, metric: str): - if metric == "average_rating": - return 4.5 - elif metric == "total_reviews": - return 312 raise NotImplementedError(f"Metric {metric} not supported") diff --git a/source/constants/__init__.py b/source/constants/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/source/constants/source_type.py b/source/constants/source_type.py new file mode 100644 index 0000000..acfa338 --- /dev/null +++ b/source/constants/source_type.py @@ -0,0 +1,7 @@ +from django.db import models + + +class SourceType(models.TextChoices): + API = "api", "API" + SCRAPER = "scraper", "Scraper" + EXTERNAL_TOOL = "external_tool", "External Tool" diff --git a/source/migrations/0002_source_code.py b/source/migrations/0002_source_code.py new file mode 100644 index 0000000..e4cdfe1 --- /dev/null +++ b/source/migrations/0002_source_code.py @@ -0,0 +1,19 @@ +# Generated by Django 5.1.7 on 2025-04-05 14:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("source", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="source", + name="code", + field=models.CharField(default="default_code", max_length=100, unique=True), + preserve_default=False, + ), + ] diff --git a/source/models.py b/source/models.py index 1a36777..e1c70a6 100644 --- a/source/models.py +++ b/source/models.py @@ -1,13 +1,10 @@ from django.db import models - -class SourceType(models.TextChoices): - API = "api", "API" - SCRAPER = "scraper", "Scraper" - EXTERNAL_TOOL = "external_tool", "External Tool" +from source.constants.source_type import SourceType class Source(models.Model): + code = models.CharField(max_length=100, unique=True) name = models.CharField(max_length=100) type = models.CharField(max_length=20, choices=SourceType.choices) url = models.URLField(null=True, blank=True) diff --git a/source/registry/__init__.py b/source/registry/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/source/registry/source_registry.py b/source/registry/source_registry.py new file mode 100644 index 0000000..3116a63 --- /dev/null +++ b/source/registry/source_registry.py @@ -0,0 +1,50 @@ +import importlib +import pkgutil +from typing import List + +import source.adapters +from source.adapters.base import SourceAdapter +from source.models import Source + + +class SourceRegistry: + def __init__(self): + self.adapters = self.load_adapters() + + def load_adapters(self) -> List[SourceAdapter]: + loaded_adapters = [] + + for _, module_name, is_pkg in pkgutil.iter_modules(source.adapters.__path__): + if is_pkg or module_name == "base": + continue + + module = importlib.import_module(f"source.adapters.{module_name}") + + for attr_name in dir(module): + attr = getattr(module, attr_name) + if ( + isinstance(attr, type) + and hasattr(attr, "code") + and hasattr(attr, "name") + and hasattr(attr, "type") + ): + instance = attr() + loaded_adapters.append(instance) + + # Registre a la base de dades si no existeix + Source.objects.get_or_create( + code=instance.code, + defaults={ + "name": instance.name, + "type": instance.type.value, + "url": getattr(instance, "url", None), + }, + ) + + return loaded_adapters + + def get_sources(self) -> List[SourceAdapter]: + return self.adapters + + def get_for_metric(self, metric: str) -> List[SourceAdapter]: + return [adapter for adapter in self.adapters if adapter.supports_metric(metric)] From c028e4bb57b3860caeb9c77bc4ccb798ecd03f3a Mon Sep 17 00:00:00 2001 From: Anyer Date: Tue, 8 Apr 2025 22:29:47 +0200 Subject: [PATCH 02/42] =?UTF-8?q?feat(tests):=20Verificar=20la=20persist?= =?UTF-8?q?=C3=A8ncia=20de=20les=20dades?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #46 --- app/tests.py | 18 +++++++++++++++ metric/tests.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++ pytest.ini | 3 +++ requirements.txt | Bin 940 -> 1082 bytes review/tests.py | 54 +++++++++++++++++++++++++++++++++++++++++++++ source/tests.py | 19 ++++++++++++++++ 6 files changed, 150 insertions(+) create mode 100644 pytest.ini diff --git a/app/tests.py b/app/tests.py index e69de29..f359d1c 100644 --- a/app/tests.py +++ b/app/tests.py @@ -0,0 +1,18 @@ +import pytest + +from app.models import App + + +@pytest.mark.django_db +def test_create_source(): + a = App.objects.create( + code="test_app", + name="Test App", + description="Descripction of the test app", + appstore_id="123456", + ) + assert a.pk is not None + assert a.code == "test_app" + assert a.name == "Test App" + assert a.description == "Descripction of the test app" + assert a.appstore_id == "123456" diff --git a/metric/tests.py b/metric/tests.py index e69de29..e0f8c2e 100644 --- a/metric/tests.py +++ b/metric/tests.py @@ -0,0 +1,56 @@ +import pytest + +from app.models import App +from metric.constants import MetricCodes +from metric.constants.value_types import MetricValueType +from metric.models import Metric, MetricValue +from metric.registry.metric_registry import register_metrics +from source.constants.source_type import SourceType +from source.models import Source + + +@pytest.mark.django_db +def test_create_metric(): + m = Metric.objects.create( + code="test_metric", + name="Test Metric", + description="Descripció de test", + value_type=MetricValueType.FLOAT, + ) + assert m.pk is not None + assert m.code == "test_metric" + + +@pytest.mark.django_db +def test_create_metric_value(): + a = App.objects.create( + code="test_app", + name="Test App", + description="Descripction of the test app", + appstore_id="123456", + ) + s = Source.objects.create( + code="test_source", + name="Test Source", + type=SourceType.API, + url="https://example.com", + ) + m = Metric.objects.create( + code="test_metric", + name="Test Metric", + description="Descripció de test", + value_type=MetricValueType.FLOAT, + ) + mv = MetricValue.objects.create(app=a, metric=m, source=s, value="123.45") + assert mv.pk is not None + assert mv.app == a + assert mv.metric == m + assert mv.source == s + assert mv.value == "123.45" + + +@pytest.mark.django_db +def test_register_metrics_creates_metrics(): + register_metrics() + assert Metric.objects.filter(code=MetricCodes.AVERAGE_RATING).exists() + assert Metric.objects.filter(code=MetricCodes.TOTAL_REVIEWS).exists() diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..7a4fb9b --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +DJANGO_SETTINGS_MODULE = config.settings +python_files = tests.py test_*.py *_tests.py diff --git a/requirements.txt b/requirements.txt index 7dab18512b160fe3fe698398a48b34ad8d6a3db3..bd549f9a947a4bc1fe44a6246da982b7ab3bda8c 100644 GIT binary patch delta 137 zcmZ3(zKdhSCPwKzhD?TJhJ1!RhBP26oxv6ejTrP83>ow$uVj2Gp2JWI6isKS1j_(L z8BBrLU~)T?p<^mTF+&Mh+yW?O43^_%;9@8MsxD!G$m%krFk~?#0?khc+5$4&1gOps MWIlu8 Date: Wed, 9 Apr 2025 17:45:08 +0200 Subject: [PATCH 03/42] feat(software-arch): Implementar la arquitectura de software completa Refs #99 --- metric/registry/definitions.py | 18 +++++++ metric/registry/metric_registry.py | 35 ------------- .../repositories}/__init__.py | 0 metric/repositories/metric_repository.py | 6 +++ metric/services/__init__.py | 0 .../services/metric_registration_service.py | 22 ++++++++ metric/strategies/base.py | 11 ---- metric/strategies/generic.py | 12 +++-- source/adapters/itunes.py | 35 +++++++------ source/definitions/__init__.py | 0 source/definitions/discovery.py | 10 ++++ source/registry/source_registry.py | 50 ------------------- source/repositories/__init__.py | 0 source/repositories/source_repository.py | 6 +++ source/services/__init__.py | 0 .../services/source_registration_service.py | 20 ++++++++ 16 files changed, 110 insertions(+), 115 deletions(-) create mode 100644 metric/registry/definitions.py delete mode 100644 metric/registry/metric_registry.py rename {source/registry => metric/repositories}/__init__.py (100%) create mode 100644 metric/repositories/metric_repository.py create mode 100644 metric/services/__init__.py create mode 100644 metric/services/metric_registration_service.py delete mode 100644 metric/strategies/base.py create mode 100644 source/definitions/__init__.py create mode 100644 source/definitions/discovery.py delete mode 100644 source/registry/source_registry.py create mode 100644 source/repositories/__init__.py create mode 100644 source/repositories/source_repository.py create mode 100644 source/services/__init__.py create mode 100644 source/services/source_registration_service.py diff --git a/metric/registry/definitions.py b/metric/registry/definitions.py new file mode 100644 index 0000000..2186549 --- /dev/null +++ b/metric/registry/definitions.py @@ -0,0 +1,18 @@ +from metric.constants import MetricCodes +from metric.constants.value_types import MetricValueType + +METRICS = [ + { + "code": MetricCodes.AVERAGE_RATING, + "name": "Average Rating", + "description": "Valoració mitjana de l'app", + "value_type": MetricValueType.FLOAT, + }, + { + "code": MetricCodes.TOTAL_REVIEWS, + "name": "Total Reviews", + "description": "Nombre total de ressenyes", + "value_type": MetricValueType.INTEGER, + }, + # Afegir més mètriques aquí +] diff --git a/metric/registry/metric_registry.py b/metric/registry/metric_registry.py deleted file mode 100644 index f986e49..0000000 --- a/metric/registry/metric_registry.py +++ /dev/null @@ -1,35 +0,0 @@ -from metric.constants import MetricCodes -from metric.constants.value_types import MetricValueType -from metric.models import Metric - -METRICS = [ - { - "code": MetricCodes.AVERAGE_RATING, - "name": "Average Rating", - "description": "Valoració mitjana de l'app", - "value_type": MetricValueType.FLOAT, - }, - { - "code": MetricCodes.TOTAL_REVIEWS, - "name": "Total Reviews", - "description": "Nombre total de ressenyes", - "value_type": MetricValueType.INTEGER, - }, - # Afegir aquí les noves -] - - -def register_metrics(): - for m in METRICS: - metric, created = Metric.objects.get_or_create( - code=m["code"], - defaults={ - "name": m["name"], - "description": m["description"], - "value_type": m["value_type"].value, - }, - ) - if created: - print(f"✅ Mètrica registrada: {m['code']}") - else: - print(f"ℹ️ Ja existia: {m['code']}") diff --git a/source/registry/__init__.py b/metric/repositories/__init__.py similarity index 100% rename from source/registry/__init__.py rename to metric/repositories/__init__.py diff --git a/metric/repositories/metric_repository.py b/metric/repositories/metric_repository.py new file mode 100644 index 0000000..1d1ef1a --- /dev/null +++ b/metric/repositories/metric_repository.py @@ -0,0 +1,6 @@ +from metric.models import Metric + + +class MetricRepository: + def get_or_create(self, code, defaults): + return Metric.objects.get_or_create(code=code, defaults=defaults) diff --git a/metric/services/__init__.py b/metric/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metric/services/metric_registration_service.py b/metric/services/metric_registration_service.py new file mode 100644 index 0000000..3f98688 --- /dev/null +++ b/metric/services/metric_registration_service.py @@ -0,0 +1,22 @@ +from metric.registry.definitions import METRICS +from metric.repositories.metric_repository import MetricRepository + + +class MetricRegistrationService: + def __init__(self, repository=None): + self.repository = repository or MetricRepository() + + def register_all(self): + for m in METRICS: + metric, created = self.repository.get_or_create( + code=m["code"], + defaults={ + "name": m["name"], + "description": m["description"], + "value_type": m["value_type"].value, + }, + ) + if created: + print(f"✅ Mètrica registrada: {m['code']}") + else: + print(f"ℹ️ Ja existia: {m['code']}") diff --git a/metric/strategies/base.py b/metric/strategies/base.py deleted file mode 100644 index 6921001..0000000 --- a/metric/strategies/base.py +++ /dev/null @@ -1,11 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List - -from metric.models import MetricValue -from source.registry.source_registry import SourceRegistry - - -class MetricStrategy(ABC): - @abstractmethod - def compute_all(self, app_id: str, registry: SourceRegistry) -> List[MetricValue]: - pass diff --git a/metric/strategies/generic.py b/metric/strategies/generic.py index 0ebe5f0..d4ec0e0 100644 --- a/metric/strategies/generic.py +++ b/metric/strategies/generic.py @@ -1,17 +1,23 @@ from datetime import datetime from metric.models import MetricValue -from source.registry.source_registry import SourceRegistry +from source.adapters.base import SourceAdapter class GenericMetricStrategy: def __init__(self, metric_code: str): self.metric_code = metric_code - def compute_all(self, app_id: str, registry: SourceRegistry) -> list[MetricValue]: + def compute_all( + self, app_id: str, adapters: list[SourceAdapter] + ) -> list[MetricValue]: results = [] - for adapter in registry.get_for_metric(self.metric_code): + for adapter in adapters: + if not adapter.supports_metric(self.metric_code): + continue + value = adapter.fetch(app_id, self.metric_code) + results.append( MetricValue( app_id=app_id, diff --git a/source/adapters/itunes.py b/source/adapters/itunes.py index 407436b..8474cc6 100644 --- a/source/adapters/itunes.py +++ b/source/adapters/itunes.py @@ -2,33 +2,36 @@ from app.models import App from metric.constants import MetricCodes -from source.adapters.base import SourceAdapter from source.constants.source_type import SourceType -class ItunesSearchAPIAdapter(SourceAdapter): +class ItunesSearchAPIAdapter: code = "itunes" name = "App Store" type = SourceType.API - url = "https://itunes.apple.com/" + url = "https://itunes.apple.com" supported_metrics = [MetricCodes.AVERAGE_RATING, MetricCodes.TOTAL_REVIEWS] def supports_metric(self, metric: str) -> bool: return metric in self.supported_metrics def fetch(self, app_name: str, metric: str): - app = App.objects.get(name=app_name) - response = requests.get(f"{self.url}lookup?id={app.appstore_id}") - data = response.json() - - if data.get("resultCount", 0) == 0 or "results" not in data: - return None # Protegeix si la resposta no té resultats + try: + app = App.objects.get(name=app_name) + except App.DoesNotExist: + return None - result = data["results"][0] + response = requests.get(f"{self.url}/lookup?id={app.appstore_id}") + if not response.ok: + return None - if metric == MetricCodes.AVERAGE_RATING: - return result.get("averageUserRating", None) - elif metric == MetricCodes.TOTAL_REVIEWS: - return result.get("userRatingCount", None) - - raise NotImplementedError(f"Metric {metric} not supported") + data = response.json() + results = data.get("results", []) + if not results: + return None + + result = results[0] + return { + MetricCodes.AVERAGE_RATING: result.get("averageUserRating"), + MetricCodes.TOTAL_REVIEWS: result.get("userRatingCount"), + }.get(metric) diff --git a/source/definitions/__init__.py b/source/definitions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/source/definitions/discovery.py b/source/definitions/discovery.py new file mode 100644 index 0000000..4aba6de --- /dev/null +++ b/source/definitions/discovery.py @@ -0,0 +1,10 @@ +from source.adapters.base import SourceAdapter + +# isort: off +import source.adapters.itunes as _ # noqa + +# isort: on + + +def discover_sources(): + return [cls() for cls in SourceAdapter.__subclasses__()] diff --git a/source/registry/source_registry.py b/source/registry/source_registry.py deleted file mode 100644 index 3116a63..0000000 --- a/source/registry/source_registry.py +++ /dev/null @@ -1,50 +0,0 @@ -import importlib -import pkgutil -from typing import List - -import source.adapters -from source.adapters.base import SourceAdapter -from source.models import Source - - -class SourceRegistry: - def __init__(self): - self.adapters = self.load_adapters() - - def load_adapters(self) -> List[SourceAdapter]: - loaded_adapters = [] - - for _, module_name, is_pkg in pkgutil.iter_modules(source.adapters.__path__): - if is_pkg or module_name == "base": - continue - - module = importlib.import_module(f"source.adapters.{module_name}") - - for attr_name in dir(module): - attr = getattr(module, attr_name) - if ( - isinstance(attr, type) - and hasattr(attr, "code") - and hasattr(attr, "name") - and hasattr(attr, "type") - ): - instance = attr() - loaded_adapters.append(instance) - - # Registre a la base de dades si no existeix - Source.objects.get_or_create( - code=instance.code, - defaults={ - "name": instance.name, - "type": instance.type.value, - "url": getattr(instance, "url", None), - }, - ) - - return loaded_adapters - - def get_sources(self) -> List[SourceAdapter]: - return self.adapters - - def get_for_metric(self, metric: str) -> List[SourceAdapter]: - return [adapter for adapter in self.adapters if adapter.supports_metric(metric)] diff --git a/source/repositories/__init__.py b/source/repositories/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/source/repositories/source_repository.py b/source/repositories/source_repository.py new file mode 100644 index 0000000..713b742 --- /dev/null +++ b/source/repositories/source_repository.py @@ -0,0 +1,6 @@ +from source.models import Source + + +class SourceRepository: + def get_or_create(self, code, defaults): + return Source.objects.get_or_create(code=code, defaults=defaults) diff --git a/source/services/__init__.py b/source/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/source/services/source_registration_service.py b/source/services/source_registration_service.py new file mode 100644 index 0000000..51215b0 --- /dev/null +++ b/source/services/source_registration_service.py @@ -0,0 +1,20 @@ +from source.definitions.discovery import discover_sources +from source.repositories.source_repository import SourceRepository + + +class SourceRegistrationService: + def __init__(self, repository=None): + self.repository = repository or SourceRepository() + + def load_and_register(self): + adapters = discover_sources() + for adapter in adapters: + self.repository.get_or_create( + code=adapter.code, + defaults={ + "name": adapter.name, + "type": adapter.type, + "url": adapter.url, + }, + ) + return adapters From 9365ff9025d4e6805a0601e239ea5d623a9dd7a6 Mon Sep 17 00:00:00 2001 From: Anyer Date: Wed, 9 Apr 2025 19:28:04 +0200 Subject: [PATCH 04/42] =?UTF-8?q?feat(all-tests):=20Repassar=20tots=20els?= =?UTF-8?q?=20units=20tests=20i=20el=20test=20d'integraci=C3=B3=20de=20tot?= =?UTF-8?q?s=20els=20components?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #47 --- .flake8 | 2 +- config/settings.py | 3 +- metric/{registry => definitions}/__init__.py | 0 .../definitions.py => definitions/metrics.py} | 0 metric/migrations/0001_initial.py | 4 +- .../services/metric_registration_service.py | 70 +++++++++++++------ metric/strategies/generic.py | 20 ++++-- metric/tests.py | 9 --- pyproject.toml | 11 +++ review/migrations/0001_initial.py | 4 +- review/models.py | 4 +- source/adapters/base.py | 6 +- source/adapters/itunes.py | 8 ++- source/definitions/discovery.py | 1 + tests/__init__.py | 0 tests/integration/__init__.py | 0 tests/integration/test_end_to_end.py | 42 +++++++++++ tests/unit/__init__.py | 0 tests/unit/test_discovery.py | 7 ++ tests/unit/test_generic_metric_strategy.py | 24 +++++++ tests/unit/test_itunes_adapter.py | 21 ++++++ .../unit/test_source_registration_service.py | 17 +++++ 22 files changed, 198 insertions(+), 55 deletions(-) rename metric/{registry => definitions}/__init__.py (100%) rename metric/{registry/definitions.py => definitions/metrics.py} (100%) create mode 100644 pyproject.toml create mode 100644 tests/__init__.py create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/test_end_to_end.py create mode 100644 tests/unit/__init__.py create mode 100644 tests/unit/test_discovery.py create mode 100644 tests/unit/test_generic_metric_strategy.py create mode 100644 tests/unit/test_itunes_adapter.py create mode 100644 tests/unit/test_source_registration_service.py diff --git a/.flake8 b/.flake8 index 05a09f3..a960d69 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,4 @@ [flake8] -max-line-length = 88 +max-line-length = 100 extend-ignore = E203, W503 exclude = .venv,venv,migrations,__pycache__,.git diff --git a/config/settings.py b/config/settings.py index 509a3a0..0da88a5 100644 --- a/config/settings.py +++ b/config/settings.py @@ -98,8 +98,7 @@ AUTH_PASSWORD_VALIDATORS = [ { - "NAME": "django.contrib.auth.password_validation." - "UserAttributeSimilarityValidator", + "NAME": "django.contrib.auth.password_validation." "UserAttributeSimilarityValidator", }, { "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", diff --git a/metric/registry/__init__.py b/metric/definitions/__init__.py similarity index 100% rename from metric/registry/__init__.py rename to metric/definitions/__init__.py diff --git a/metric/registry/definitions.py b/metric/definitions/metrics.py similarity index 100% rename from metric/registry/definitions.py rename to metric/definitions/metrics.py diff --git a/metric/migrations/0001_initial.py b/metric/migrations/0001_initial.py index 8d1c2f4..146a52d 100644 --- a/metric/migrations/0001_initial.py +++ b/metric/migrations/0001_initial.py @@ -59,9 +59,7 @@ class Migration(migrations.Migration): ("retrieved_at", models.DateTimeField(auto_now_add=True)), ( "app", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, to="app.app" - ), + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="app.app"), ), ( "metric", diff --git a/metric/services/metric_registration_service.py b/metric/services/metric_registration_service.py index 3f98688..03224c2 100644 --- a/metric/services/metric_registration_service.py +++ b/metric/services/metric_registration_service.py @@ -1,22 +1,48 @@ -from metric.registry.definitions import METRICS -from metric.repositories.metric_repository import MetricRepository - - -class MetricRegistrationService: - def __init__(self, repository=None): - self.repository = repository or MetricRepository() - - def register_all(self): - for m in METRICS: - metric, created = self.repository.get_or_create( - code=m["code"], - defaults={ - "name": m["name"], - "description": m["description"], - "value_type": m["value_type"].value, - }, - ) - if created: - print(f"✅ Mètrica registrada: {m['code']}") - else: - print(f"ℹ️ Ja existia: {m['code']}") +from datetime import datetime + +import pytest + +from app.models import App +from metric.models import Metric +from metric.strategies.generic import GenericMetricStrategy +from source.constants.source_type import SourceType +from source.models import Source + + +@pytest.mark.django_db +def test_generic_metric_strategy_computes_values(): + # 1. Crear els objectes necessaris a la BD + app = App.objects.create(name="Test App") + metric = Metric.objects.create(code="average_rating", name="Average Rating", value_type="float") + source = Source.objects.create( + code="fake", name="Fake Source", type=SourceType.API, url="http://fake" + ) + + # 2. Adapter fals + class FakeAdapter: + code = "fake" + name = "Fake Source" + type = SourceType.API + url = "http://fake" + supported_metrics = ["average_rating"] + + def supports_metric(self, metric: str) -> bool: + return True + + def fetch(self, app_id: str, metric: str): + return 4.8 + + adapter = FakeAdapter() + + # 3. Executar l'estratègia + strategy = GenericMetricStrategy("average_rating") + result = strategy.compute_all(app.id, [adapter]) + + # 4. Validació + assert len(result) == 1 + value = result[0] + assert value.app == app + assert value.metric == metric + assert value.source == source + assert value.value == 4.8 + assert isinstance(value.retrieved_at, datetime) diff --git a/metric/strategies/generic.py b/metric/strategies/generic.py index d4ec0e0..c4b2825 100644 --- a/metric/strategies/generic.py +++ b/metric/strategies/generic.py @@ -1,28 +1,34 @@ from datetime import datetime -from metric.models import MetricValue +from app.models import App +from metric.models import Metric, MetricValue from source.adapters.base import SourceAdapter +from source.models import Source class GenericMetricStrategy: def __init__(self, metric_code: str): self.metric_code = metric_code - def compute_all( - self, app_id: str, adapters: list[SourceAdapter] - ) -> list[MetricValue]: + def compute_all(self, app_id: str, adapters: list[SourceAdapter]) -> list[MetricValue]: results = [] + + app = App.objects.get(id=app_id) + metric = Metric.objects.get(code=self.metric_code) + for adapter in adapters: if not adapter.supports_metric(self.metric_code): continue value = adapter.fetch(app_id, self.metric_code) + source = Source.objects.get(code=adapter.code) + results.append( MetricValue( - app_id=app_id, - metric_code=self.metric_code, - source_name=adapter.__class__.__name__, + app=app, + metric=metric, + source=source, value=value, retrieved_at=datetime.now(), ) diff --git a/metric/tests.py b/metric/tests.py index e0f8c2e..7fe1653 100644 --- a/metric/tests.py +++ b/metric/tests.py @@ -1,10 +1,8 @@ import pytest from app.models import App -from metric.constants import MetricCodes from metric.constants.value_types import MetricValueType from metric.models import Metric, MetricValue -from metric.registry.metric_registry import register_metrics from source.constants.source_type import SourceType from source.models import Source @@ -47,10 +45,3 @@ def test_create_metric_value(): assert mv.metric == m assert mv.source == s assert mv.value == "123.45" - - -@pytest.mark.django_db -def test_register_metrics_creates_metrics(): - register_metrics() - assert Metric.objects.filter(code=MetricCodes.AVERAGE_RATING).exists() - assert Metric.objects.filter(code=MetricCodes.TOTAL_REVIEWS).exists() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..c8d1fe8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,11 @@ +[tool.black] +line-length = 100 +target-version = ['py312'] + +[tool.isort] +profile = "black" +line_length = 100 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true diff --git a/review/migrations/0001_initial.py b/review/migrations/0001_initial.py index a9d3f5a..fcecd63 100644 --- a/review/migrations/0001_initial.py +++ b/review/migrations/0001_initial.py @@ -83,9 +83,7 @@ class Migration(migrations.Migration): ("date", models.DateTimeField()), ( "app", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, to="app.app" - ), + models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="app.app"), ), ], ), diff --git a/review/models.py b/review/models.py index 9077805..eef4159 100644 --- a/review/models.py +++ b/review/models.py @@ -37,9 +37,7 @@ class Review(models.Model): ) polarity = models.CharField(max_length=10, choices=ReviewPolarity.choices) type = models.CharField(max_length=20, choices=ReviewType.choices) - topic = models.CharField( - max_length=30, choices=ReviewTopic.choices, null=True, blank=True - ) + topic = models.CharField(max_length=30, choices=ReviewTopic.choices, null=True, blank=True) date = models.DateTimeField() def __str__(self): diff --git a/source/adapters/base.py b/source/adapters/base.py index 12bd8e5..fe924f0 100644 --- a/source/adapters/base.py +++ b/source/adapters/base.py @@ -1,15 +1,17 @@ -from typing import Protocol +from abc import ABC, abstractmethod from source.constants.source_type import SourceType -class SourceAdapter(Protocol): +class SourceAdapter(ABC): code: str name: str type: SourceType url: str | None supported_metrics: list[str] + @abstractmethod def supports_metric(self, metric: str) -> bool: ... + @abstractmethod def fetch(self, app_id: str, metric: str) -> str: ... diff --git a/source/adapters/itunes.py b/source/adapters/itunes.py index 8474cc6..44d1b34 100644 --- a/source/adapters/itunes.py +++ b/source/adapters/itunes.py @@ -2,10 +2,11 @@ from app.models import App from metric.constants import MetricCodes +from source.adapters.base import SourceAdapter from source.constants.source_type import SourceType -class ItunesSearchAPIAdapter: +class ItunesSearchAPIAdapter(SourceAdapter): code = "itunes" name = "App Store" type = SourceType.API @@ -15,9 +16,9 @@ class ItunesSearchAPIAdapter: def supports_metric(self, metric: str) -> bool: return metric in self.supported_metrics - def fetch(self, app_name: str, metric: str): + def fetch(self, app_id: str, metric: str): try: - app = App.objects.get(name=app_name) + app = App.objects.get(id=app_id) except App.DoesNotExist: return None @@ -27,6 +28,7 @@ def fetch(self, app_name: str, metric: str): data = response.json() results = data.get("results", []) + if not results: return None diff --git a/source/definitions/discovery.py b/source/definitions/discovery.py index 4aba6de..2150cd5 100644 --- a/source/definitions/discovery.py +++ b/source/definitions/discovery.py @@ -7,4 +7,5 @@ def discover_sources(): + print(SourceAdapter.__subclasses__()) return [cls() for cls in SourceAdapter.__subclasses__()] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py new file mode 100644 index 0000000..2414d8a --- /dev/null +++ b/tests/integration/test_end_to_end.py @@ -0,0 +1,42 @@ +import pytest + +from app.models import App +from metric.models import Metric +from metric.strategies.generic import GenericMetricStrategy +from source.constants.source_type import SourceType +from source.models import Source +from source.services.source_registration_service import SourceRegistrationService + + +@pytest.mark.django_db +def test_full_flow_with_real_adapter(): + # Setup: crear App, Metric y Source reals + app = App.objects.create( + code="discord", + name="Discord", + description="App de comunicació global", + appstore_id="985746746", + ) + metric = Metric.objects.create(code="average_rating", name="Average Rating", value_type="float") + source = Source.objects.create( + code="itunes", + name="iTunes Search API", + type=SourceType.API, + url="https://itunes.apple.com", + ) + + # Adapter real (ja ha de formar part del projecte i registrar-se automàticament) + adapters = SourceRegistrationService().load_and_register() + + strategy = GenericMetricStrategy("average_rating") + values = strategy.compute_all(app.id, adapters) + + for v in values: + print(f"metric={v.metric.code}, app={v.app.code}, source={v.source.code}, value={v.value}") + + assert isinstance(values, list) + assert any( + v.metric == metric and v.app == app and v.source == source + for v in values + if v.value is not None + ) diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_discovery.py b/tests/unit/test_discovery.py new file mode 100644 index 0000000..69f8753 --- /dev/null +++ b/tests/unit/test_discovery.py @@ -0,0 +1,7 @@ +from source.adapters.itunes import ItunesSearchAPIAdapter +from source.definitions.discovery import discover_sources + + +def test_discover_sources_returns_adapters(): + adapters = discover_sources() + assert any(isinstance(a, ItunesSearchAPIAdapter) for a in adapters) diff --git a/tests/unit/test_generic_metric_strategy.py b/tests/unit/test_generic_metric_strategy.py new file mode 100644 index 0000000..8b267d4 --- /dev/null +++ b/tests/unit/test_generic_metric_strategy.py @@ -0,0 +1,24 @@ +from metric.strategies.generic import GenericMetricStrategy + + +class FakeAdapter: + code = "fake" + name = "Fake Source" + type = "api" + url = "http://fake" + supported_metrics = ["average_rating"] + + def supports_metric(self, metric): + return True + + def fetch(self, app_id, metric): + return 4.8 + + +def test_generic_metric_strategy_computes_values(): + adapter = FakeAdapter() + strategy = GenericMetricStrategy("average_rating") + result = strategy.compute_all("some_app", [adapter]) + + assert len(result) == 1 + assert result[0].value == 4.8 diff --git a/tests/unit/test_itunes_adapter.py b/tests/unit/test_itunes_adapter.py new file mode 100644 index 0000000..e214e47 --- /dev/null +++ b/tests/unit/test_itunes_adapter.py @@ -0,0 +1,21 @@ +from unittest.mock import patch + +from metric.constants import MetricCodes +from source.adapters.itunes import ItunesSearchAPIAdapter + + +@patch("source.adapters.itunes.App.objects.get") +@patch("source.adapters.itunes.requests.get") +def test_itunes_fetch(mock_get, mock_app_get): + mock_app_get.return_value.appstore_id = "123" + mock_get.return_value.ok = True + mock_get.return_value.json.return_value = { + "resultCount": 1, + "results": [{"averageUserRating": 4.5, "userRatingCount": 1000}], + } + + adapter = ItunesSearchAPIAdapter() + + assert adapter.supports_metric(MetricCodes.AVERAGE_RATING) + assert adapter.fetch("my_app", MetricCodes.AVERAGE_RATING) == 4.5 + assert adapter.fetch("my_app", MetricCodes.TOTAL_REVIEWS) == 1000 diff --git a/tests/unit/test_source_registration_service.py b/tests/unit/test_source_registration_service.py new file mode 100644 index 0000000..7cea782 --- /dev/null +++ b/tests/unit/test_source_registration_service.py @@ -0,0 +1,17 @@ +from source.services.source_registration_service import SourceRegistrationService + + +class FakeRepository: + def __init__(self): + self.registered = [] + + def get_or_create(self, code, defaults): + self.registered.append((code, defaults)) + return None + + +def test_source_registration_service_registers_adapters(): + repo = FakeRepository() + service = SourceRegistrationService(repository=repo) + adapters = service.load_and_register() + assert len(repo.registered) == len(adapters) From 0350bfb814e963e649ca3fa1186ececd285c026e Mon Sep 17 00:00:00 2001 From: Anyer Date: Wed, 9 Apr 2025 19:31:25 +0200 Subject: [PATCH 05/42] =?UTF-8?q?feat(all-tests):=20Repassar=20tots=20els?= =?UTF-8?q?=20units=20tests=20i=20el=20test=20d'integraci=C3=B3=20de=20tot?= =?UTF-8?q?s=20els=20components?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #47 --- tests/unit/test_generic_metric_strategy.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_generic_metric_strategy.py b/tests/unit/test_generic_metric_strategy.py index 8b267d4..352fa1c 100644 --- a/tests/unit/test_generic_metric_strategy.py +++ b/tests/unit/test_generic_metric_strategy.py @@ -1,4 +1,10 @@ +import pytest + +from app.models import App +from metric.models import Metric from metric.strategies.generic import GenericMetricStrategy +from source.constants.source_type import SourceType +from source.models import Source class FakeAdapter: @@ -15,10 +21,21 @@ def fetch(self, app_id, metric): return 4.8 +@pytest.mark.django_db def test_generic_metric_strategy_computes_values(): + # Crear objectes de BD necessaris per a la prova + app = App.objects.create(name="Test App", code="test", appstore_id="123456") + metric = Metric.objects.create(code="average_rating", name="Average Rating", value_type="float") + source = Source.objects.create( + code="fake", name="Fake Source", type=SourceType.API, url="http://fake" + ) + adapter = FakeAdapter() strategy = GenericMetricStrategy("average_rating") - result = strategy.compute_all("some_app", [adapter]) + result = strategy.compute_all(app.id, [adapter]) assert len(result) == 1 assert result[0].value == 4.8 + assert result[0].metric == metric + assert result[0].app == app + assert result[0].source == source From 9dcaec224686d68d0705cb0662d36be1613a1655 Mon Sep 17 00:00:00 2001 From: Anyer Date: Thu, 10 Apr 2025 19:22:20 +0200 Subject: [PATCH 06/42] Refactor: move discover_sources into SourceRegistrationService as static method --- review/tests.py | 11 +++++------ source/definitions/discovery.py | 11 ----------- source/repositories/source_repository.py | 3 ++- source/services/source_registration_service.py | 13 +++++++++++-- tests/unit/test_discovery.py | 7 ------- tests/unit/test_source_registration_service.py | 5 +++++ 6 files changed, 23 insertions(+), 27 deletions(-) delete mode 100644 source/definitions/discovery.py delete mode 100644 tests/unit/test_discovery.py diff --git a/review/tests.py b/review/tests.py index 4191591..9f81184 100644 --- a/review/tests.py +++ b/review/tests.py @@ -1,14 +1,13 @@ -from datetime import datetime - import pytest +from django.utils import timezone from app.models import App from review.models import Review, ReviewPolarity, ReviewTopic, ReviewType @pytest.mark.django_db -def test_create_source_no_values(): - now = datetime.now() +def test_create_review_no_values(): + now = timezone.now() a = App.objects.create( code="test_app", name="Test App", @@ -27,8 +26,8 @@ def test_create_source_no_values(): @pytest.mark.django_db -def test_create_source_with_values(): - now = datetime.now() +def test_create_review_with_values(): + now = timezone.now() a = App.objects.create( code="test_app", name="Test App", diff --git a/source/definitions/discovery.py b/source/definitions/discovery.py deleted file mode 100644 index 2150cd5..0000000 --- a/source/definitions/discovery.py +++ /dev/null @@ -1,11 +0,0 @@ -from source.adapters.base import SourceAdapter - -# isort: off -import source.adapters.itunes as _ # noqa - -# isort: on - - -def discover_sources(): - print(SourceAdapter.__subclasses__()) - return [cls() for cls in SourceAdapter.__subclasses__()] diff --git a/source/repositories/source_repository.py b/source/repositories/source_repository.py index 713b742..7ae5108 100644 --- a/source/repositories/source_repository.py +++ b/source/repositories/source_repository.py @@ -2,5 +2,6 @@ class SourceRepository: - def get_or_create(self, code, defaults): + @staticmethod + def get_or_create(code, defaults): return Source.objects.get_or_create(code=code, defaults=defaults) diff --git a/source/services/source_registration_service.py b/source/services/source_registration_service.py index 51215b0..a9d7e58 100644 --- a/source/services/source_registration_service.py +++ b/source/services/source_registration_service.py @@ -1,13 +1,22 @@ -from source.definitions.discovery import discover_sources +from source.adapters.base import SourceAdapter from source.repositories.source_repository import SourceRepository +# isort: off +import source.adapters.itunes as _ # noqa + +# isort: on + class SourceRegistrationService: def __init__(self, repository=None): self.repository = repository or SourceRepository() + @staticmethod + def _discover_sources(): + return [cls() for cls in SourceAdapter.__subclasses__()] + def load_and_register(self): - adapters = discover_sources() + adapters = self._discover_sources() for adapter in adapters: self.repository.get_or_create( code=adapter.code, diff --git a/tests/unit/test_discovery.py b/tests/unit/test_discovery.py deleted file mode 100644 index 69f8753..0000000 --- a/tests/unit/test_discovery.py +++ /dev/null @@ -1,7 +0,0 @@ -from source.adapters.itunes import ItunesSearchAPIAdapter -from source.definitions.discovery import discover_sources - - -def test_discover_sources_returns_adapters(): - adapters = discover_sources() - assert any(isinstance(a, ItunesSearchAPIAdapter) for a in adapters) diff --git a/tests/unit/test_source_registration_service.py b/tests/unit/test_source_registration_service.py index 7cea782..fd15b64 100644 --- a/tests/unit/test_source_registration_service.py +++ b/tests/unit/test_source_registration_service.py @@ -15,3 +15,8 @@ def test_source_registration_service_registers_adapters(): service = SourceRegistrationService(repository=repo) adapters = service.load_and_register() assert len(repo.registered) == len(adapters) + + +def test_discover_sources_works(): + adapters = SourceRegistrationService._discover_sources() + assert len(adapters) > 0 From 37398b71836fce01355ffc6b0b8f11de02d1e26e Mon Sep 17 00:00:00 2001 From: Anyer Date: Thu, 10 Apr 2025 22:33:35 +0200 Subject: [PATCH 07/42] Refactor: Borrar el directori definitions de sources. --- source/definitions/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 source/definitions/__init__.py diff --git a/source/definitions/__init__.py b/source/definitions/__init__.py deleted file mode 100644 index e69de29..0000000 From 16b60fab17c17a28365a32e428f27f27b40b24bf Mon Sep 17 00:00:00 2001 From: Anyer Date: Thu, 10 Apr 2025 22:52:28 +0200 Subject: [PATCH 08/42] =?UTF-8?q?Hotfix:=20Arreglar=20el=20servei=20de=20m?= =?UTF-8?q?etric,=20tamb=C3=A9=20crear=20el=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metric/repositories/metric_repository.py | 3 +- .../services/metric_registration_service.py | 70 ++++++------------- tests/unit/test_generic_metric_strategy.py | 47 +++++++------ .../unit/test_metric_registration_service.py | 24 +++++++ 4 files changed, 75 insertions(+), 69 deletions(-) create mode 100644 tests/unit/test_metric_registration_service.py diff --git a/metric/repositories/metric_repository.py b/metric/repositories/metric_repository.py index 1d1ef1a..dec911a 100644 --- a/metric/repositories/metric_repository.py +++ b/metric/repositories/metric_repository.py @@ -2,5 +2,6 @@ class MetricRepository: - def get_or_create(self, code, defaults): + @staticmethod + def get_or_create(code, defaults): return Metric.objects.get_or_create(code=code, defaults=defaults) diff --git a/metric/services/metric_registration_service.py b/metric/services/metric_registration_service.py index 03224c2..fd979c7 100644 --- a/metric/services/metric_registration_service.py +++ b/metric/services/metric_registration_service.py @@ -1,48 +1,22 @@ -from datetime import datetime - -import pytest - -from app.models import App -from metric.models import Metric -from metric.strategies.generic import GenericMetricStrategy -from source.constants.source_type import SourceType -from source.models import Source - - -@pytest.mark.django_db -def test_generic_metric_strategy_computes_values(): - # 1. Crear els objectes necessaris a la BD - app = App.objects.create(name="Test App") - metric = Metric.objects.create(code="average_rating", name="Average Rating", value_type="float") - source = Source.objects.create( - code="fake", name="Fake Source", type=SourceType.API, url="http://fake" - ) - - # 2. Adapter fals - class FakeAdapter: - code = "fake" - name = "Fake Source" - type = SourceType.API - url = "http://fake" - supported_metrics = ["average_rating"] - - def supports_metric(self, metric: str) -> bool: - return True - - def fetch(self, app_id: str, metric: str): - return 4.8 - - adapter = FakeAdapter() - - # 3. Executar l'estratègia - strategy = GenericMetricStrategy("average_rating") - result = strategy.compute_all(app.id, [adapter]) - - # 4. Validació - assert len(result) == 1 - value = result[0] - assert value.app == app - assert value.metric == metric - assert value.source == source - assert value.value == 4.8 - assert isinstance(value.retrieved_at, datetime) +from metric.definitions.metrics import METRICS +from metric.repositories.metric_repository import MetricRepository + + +class MetricRegistrationService: + def __init__(self, repository=None): + self.repository = repository or MetricRepository() + + def register_all(self): + for m in METRICS: + metric, created = self.repository.get_or_create( + code=m["code"], + defaults={ + "name": m["name"], + "description": m["description"], + "value_type": m["value_type"].value, + }, + ) + if created: + print(f"✅ Mètrica registrada: {m['code']}") + else: + print(f"ℹ️ Ja existia: {m['code']}") diff --git a/tests/unit/test_generic_metric_strategy.py b/tests/unit/test_generic_metric_strategy.py index 352fa1c..03224c2 100644 --- a/tests/unit/test_generic_metric_strategy.py +++ b/tests/unit/test_generic_metric_strategy.py @@ -1,3 +1,5 @@ +from datetime import datetime + import pytest from app.models import App @@ -7,35 +9,40 @@ from source.models import Source -class FakeAdapter: - code = "fake" - name = "Fake Source" - type = "api" - url = "http://fake" - supported_metrics = ["average_rating"] - - def supports_metric(self, metric): - return True - - def fetch(self, app_id, metric): - return 4.8 - - @pytest.mark.django_db def test_generic_metric_strategy_computes_values(): - # Crear objectes de BD necessaris per a la prova - app = App.objects.create(name="Test App", code="test", appstore_id="123456") + # 1. Crear els objectes necessaris a la BD + app = App.objects.create(name="Test App") metric = Metric.objects.create(code="average_rating", name="Average Rating", value_type="float") source = Source.objects.create( code="fake", name="Fake Source", type=SourceType.API, url="http://fake" ) + # 2. Adapter fals + class FakeAdapter: + code = "fake" + name = "Fake Source" + type = SourceType.API + url = "http://fake" + supported_metrics = ["average_rating"] + + def supports_metric(self, metric: str) -> bool: + return True + + def fetch(self, app_id: str, metric: str): + return 4.8 + adapter = FakeAdapter() + + # 3. Executar l'estratègia strategy = GenericMetricStrategy("average_rating") result = strategy.compute_all(app.id, [adapter]) + # 4. Validació assert len(result) == 1 - assert result[0].value == 4.8 - assert result[0].metric == metric - assert result[0].app == app - assert result[0].source == source + value = result[0] + assert value.app == app + assert value.metric == metric + assert value.source == source + assert value.value == 4.8 + assert isinstance(value.retrieved_at, datetime) diff --git a/tests/unit/test_metric_registration_service.py b/tests/unit/test_metric_registration_service.py new file mode 100644 index 0000000..39d4da9 --- /dev/null +++ b/tests/unit/test_metric_registration_service.py @@ -0,0 +1,24 @@ +from metric.definitions.metrics import METRICS +from metric.services.metric_registration_service import MetricRegistrationService + + +class FakeMetricRepository: + def __init__(self): + self.created = [] + + def get_or_create(self, code, defaults): + # Simula que totes les mètriques són noves + self.created.append((code, defaults)) + return {"code": code, **defaults}, True + + +def test_metric_registration_service_registers_all_metrics(): + fake_repo = FakeMetricRepository() + service = MetricRegistrationService(repository=fake_repo) + + service.register_all() + + assert len(fake_repo.created) == len(METRICS) + + codes = [c for c, _ in fake_repo.created] + assert "average_rating" in codes From 84bba57d53d523169401107815c2afabedce9cf7 Mon Sep 17 00:00:00 2001 From: Anyer Date: Wed, 16 Apr 2025 18:43:07 +0200 Subject: [PATCH 09/42] =?UTF-8?q?feat(news):=20Dissenyar=20i=20implementar?= =?UTF-8?q?=20el=20connector=20a=20l=E2=80=99API=20corresponent=20i=20Crea?= =?UTF-8?q?r=20el=20connector=20dins=20l=E2=80=99app=20sources=20amb=20sup?= =?UTF-8?q?ort=20per=20fetch=20puntual?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #54, #55 --- metric/constants/__init__.py | 1 + metric/definitions/metrics.py | 6 ++++ source/adapters/base.py | 2 ++ source/adapters/news.py | 54 +++++++++++++++++++++++++++++++++ source/apps.py | 4 +++ tests/unit/test_news_adapter.py | 30 ++++++++++++++++++ 6 files changed, 97 insertions(+) create mode 100644 source/adapters/news.py create mode 100644 tests/unit/test_news_adapter.py diff --git a/metric/constants/__init__.py b/metric/constants/__init__.py index e4114a4..4641718 100644 --- a/metric/constants/__init__.py +++ b/metric/constants/__init__.py @@ -1,3 +1,4 @@ class MetricCodes: AVERAGE_RATING = "average_rating" TOTAL_REVIEWS = "total_reviews" + DAILY_NEWS_BLOG_MENTIONS = "daily_news_blog_mentions" diff --git a/metric/definitions/metrics.py b/metric/definitions/metrics.py index 2186549..8421ada 100644 --- a/metric/definitions/metrics.py +++ b/metric/definitions/metrics.py @@ -14,5 +14,11 @@ "description": "Nombre total de ressenyes", "value_type": MetricValueType.INTEGER, }, + { + "code": MetricCodes.DAILY_NEWS_BLOG_MENTIONS, + "name": "Daily News Blog Mentions", + "description": "Nombre de mencions diàries a blogs de notícies", + "value_type": MetricValueType.INTEGER, + }, # Afegir més mètriques aquí ] diff --git a/source/adapters/base.py b/source/adapters/base.py index fe924f0..b5fc05e 100644 --- a/source/adapters/base.py +++ b/source/adapters/base.py @@ -1,4 +1,5 @@ from abc import ABC, abstractmethod +from typing import Optional from source.constants.source_type import SourceType @@ -9,6 +10,7 @@ class SourceAdapter(ABC): type: SourceType url: str | None supported_metrics: list[str] + api_key: Optional[str] = None @abstractmethod def supports_metric(self, metric: str) -> bool: ... diff --git a/source/adapters/news.py b/source/adapters/news.py new file mode 100644 index 0000000..3f3b825 --- /dev/null +++ b/source/adapters/news.py @@ -0,0 +1,54 @@ +import os +from datetime import datetime + +import requests + +from app.models import App +from metric.constants import MetricCodes +from source.adapters.base import SourceAdapter +from source.constants.source_type import SourceType + + +class NewsAPIAdapter(SourceAdapter): + code = "news" + name = "News API" + type = SourceType.API + url = "https://newsapi.org/v2" + supported_metrics = [MetricCodes.DAILY_NEWS_BLOG_MENTIONS] + + def __init__(self, api_key=None): + self.api_key = api_key or os.environ.get("NEWSAPI_KEY") + + def supports_metric(self, metric: str) -> bool: + return metric in self.supported_metrics + + def fetch(self, app_id: str, metric: str): + try: + app = App.objects.get(id=app_id) + except App.DoesNotExist: + return None + + if not self.api_key: + return None + + today = datetime.today().strftime("%Y-%m-%d") + params = { + "q": app.code, + "from": today, + "to": today, + "sortBy": "publishedAt", + "language": "en", + "apiKey": self.api_key, + } + + response = requests.get(f"{self.url}/everything", params=params) + if not response.ok: + return None + + data = response.json() + print(data) + articles = data.get("articles", []) + + return { + MetricCodes.DAILY_NEWS_BLOG_MENTIONS: len(articles), + }.get(metric) diff --git a/source/apps.py b/source/apps.py index 06d886d..4bbfa3d 100644 --- a/source/apps.py +++ b/source/apps.py @@ -4,3 +4,7 @@ class SourceConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "source" + + # def ready(self): + # from source.services.source_registration_service import SourceRegistrationService + # SourceRegistrationService().load_and_register() diff --git a/tests/unit/test_news_adapter.py b/tests/unit/test_news_adapter.py new file mode 100644 index 0000000..0667f9e --- /dev/null +++ b/tests/unit/test_news_adapter.py @@ -0,0 +1,30 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from app.models import App +from metric.constants import MetricCodes +from source.adapters.news import NewsAPIAdapter + + +@pytest.mark.django_db +@patch("source.adapters.news.requests.get") +def test_news_api_adapter_fetch(mock_get): + app = App.objects.create( + code="discord", + name="Discord", + appstore_id="985746746", + playstore_id="com.discord", + description="App de comunicació global", + ) + + mock_response = MagicMock() + mock_response.ok = True + mock_response.json.return_value = {"articles": [{}, {}, {}]} + mock_get.return_value = mock_response + + adapter = NewsAPIAdapter(api_key="fake_key") + + result = adapter.fetch(app.id, MetricCodes.DAILY_NEWS_BLOG_MENTIONS) + + assert result == 3 From 08b3db00481702d822f45ddc944668d5abcc0e49 Mon Sep 17 00:00:00 2001 From: Anyer Date: Wed, 16 Apr 2025 18:58:50 +0200 Subject: [PATCH 10/42] feat(news): Guardar les dades obtingudes a la base de dades Refs #56 --- metric/models.py | 3 +-- source/migrations/0003_source_metrics.py | 19 +++++++++++++++++++ source/models.py | 1 + 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 source/migrations/0003_source_metrics.py diff --git a/metric/models.py b/metric/models.py index 1f9269b..602c5b5 100644 --- a/metric/models.py +++ b/metric/models.py @@ -2,7 +2,6 @@ from app.models import App from metric.constants.value_types import MetricValueType -from source.models import Source class Metric(models.Model): @@ -18,7 +17,7 @@ def __str__(self): class MetricValue(models.Model): app = models.ForeignKey(App, on_delete=models.CASCADE) metric = models.ForeignKey(Metric, on_delete=models.CASCADE) - source = models.ForeignKey(Source, on_delete=models.CASCADE) + source = models.ForeignKey("source.Source", on_delete=models.CASCADE) value = models.CharField(max_length=255) retrieved_at = models.DateTimeField(auto_now_add=True) diff --git a/source/migrations/0003_source_metrics.py b/source/migrations/0003_source_metrics.py new file mode 100644 index 0000000..f261457 --- /dev/null +++ b/source/migrations/0003_source_metrics.py @@ -0,0 +1,19 @@ +# Generated by Django 5.1.7 on 2025-04-16 16:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("metric", "0003_alter_metric_code"), + ("source", "0002_source_code"), + ] + + operations = [ + migrations.AddField( + model_name="source", + name="metrics", + field=models.ManyToManyField(related_name="sources", to="metric.metric"), + ), + ] diff --git a/source/models.py b/source/models.py index e1c70a6..1c3a1b5 100644 --- a/source/models.py +++ b/source/models.py @@ -8,6 +8,7 @@ class Source(models.Model): name = models.CharField(max_length=100) type = models.CharField(max_length=20, choices=SourceType.choices) url = models.URLField(null=True, blank=True) + metrics = models.ManyToManyField("metric.Metric", related_name="sources") def __str__(self): return self.name From ac156f51ec0e0df0318e3129cd87a469e4887f95 Mon Sep 17 00:00:00 2001 From: Anyer Date: Wed, 16 Apr 2025 19:24:54 +0200 Subject: [PATCH 11/42] =?UTF-8?q?feat(news):=20Creaci=C3=B3=20de=20la=20co?= =?UTF-8?q?mmanda=20poll=5Fmetrics=20per=20a=20recullir-les=20di=C3=A0riam?= =?UTF-8?q?ent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #112 --- metric/management/__init__.py | 0 metric/management/commands/__init__.py | 0 metric/management/commands/poll_metrics.py | 47 +++++++++++++++++++ source/adapters/news.py | 1 - .../services/source_registration_service.py | 1 + 5 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 metric/management/__init__.py create mode 100644 metric/management/commands/__init__.py create mode 100644 metric/management/commands/poll_metrics.py diff --git a/metric/management/__init__.py b/metric/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metric/management/commands/__init__.py b/metric/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metric/management/commands/poll_metrics.py b/metric/management/commands/poll_metrics.py new file mode 100644 index 0000000..55be67e --- /dev/null +++ b/metric/management/commands/poll_metrics.py @@ -0,0 +1,47 @@ +from django.core.management.base import BaseCommand +from django.utils import timezone + +from app.models import App +from metric.models import Metric +from metric.strategies.generic import GenericMetricStrategy +from source.services.source_registration_service import SourceRegistrationService + + +class Command(BaseCommand): + help = "Recull mètriques per a totes les apps i fonts disponibles" + + def handle(self, *args, **options): + print("[🔁] Iniciant recollida de mètriques diàries...") + + adapters = SourceRegistrationService().load_and_register() + apps = App.objects.all() + metrics = Metric.objects.prefetch_related("sources").all() + + for metric in metrics: + print(f"📌 Processant mètrica: {metric.code}") + strategy = GenericMetricStrategy(metric.code) + source_codes = metric.sources.values_list("code", flat=True) + matching_adapters = [a for a in adapters if a.code in source_codes] + + for app in apps: + print(f" 🔍 App: {app.code}") + values = strategy.compute_all(app.id, matching_adapters) + + for value in values: + if value.value is None: + print(f" ⚠️ {metric.code} ({value.source}) no ha retornat valor.") + continue + + exists = value.__class__.objects.filter( + app=app, + metric=metric, + source=value.source, + retrieved_at__date=timezone.now().date(), + ).exists() + + if not exists: + value.save() + print(f" ✅ Valor guardat: {value.value} des de {value.source}") + else: + print(f" ℹ️ Ja existia: {metric.code} per {app.code} via {value.source}") + print("✅ Procés completat.") diff --git a/source/adapters/news.py b/source/adapters/news.py index 3f3b825..817c28c 100644 --- a/source/adapters/news.py +++ b/source/adapters/news.py @@ -46,7 +46,6 @@ def fetch(self, app_id: str, metric: str): return None data = response.json() - print(data) articles = data.get("articles", []) return { diff --git a/source/services/source_registration_service.py b/source/services/source_registration_service.py index a9d7e58..f77fa0d 100644 --- a/source/services/source_registration_service.py +++ b/source/services/source_registration_service.py @@ -3,6 +3,7 @@ # isort: off import source.adapters.itunes as _ # noqa +import source.adapters.news as _ # noqa # isort: on From ad8e555c451da2edc2284f82e34b4a3e821adf6c Mon Sep 17 00:00:00 2001 From: Anyer Date: Fri, 18 Apr 2025 14:17:25 +0200 Subject: [PATCH 12/42] =?UTF-8?q?feat(reddit):=20Dissenyar=20i=20implement?= =?UTF-8?q?ar=20el=20connector=20a=20l=E2=80=99API=20corresponent=20(Reddi?= =?UTF-8?q?t)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #113 --- metric/constants/__init__.py | 1 + metric/definitions/metrics.py | 6 +++ requirements.txt | Bin 1082 -> 1240 bytes source/adapters/base.py | 2 +- source/adapters/itunes.py | 2 +- source/adapters/news.py | 2 +- source/adapters/reddit.py | 47 ++++++++++++++++++ .../services/source_registration_service.py | 1 + 8 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 source/adapters/reddit.py diff --git a/metric/constants/__init__.py b/metric/constants/__init__.py index 4641718..582ff47 100644 --- a/metric/constants/__init__.py +++ b/metric/constants/__init__.py @@ -2,3 +2,4 @@ class MetricCodes: AVERAGE_RATING = "average_rating" TOTAL_REVIEWS = "total_reviews" DAILY_NEWS_BLOG_MENTIONS = "daily_news_blog_mentions" + DAILY_REDDIT_MENTIONS = "daily_reddit_mentions" diff --git a/metric/definitions/metrics.py b/metric/definitions/metrics.py index 8421ada..855f06d 100644 --- a/metric/definitions/metrics.py +++ b/metric/definitions/metrics.py @@ -20,5 +20,11 @@ "description": "Nombre de mencions diàries a blogs de notícies", "value_type": MetricValueType.INTEGER, }, + { + "code": MetricCodes.DAILY_REDDIT_MENTIONS, + "name": "Daily Reddit Mentions", + "description": "Nombre de mencions diàries a Reddit", + "value_type": MetricValueType.INTEGER, + }, # Afegir més mètriques aquí ] diff --git a/requirements.txt b/requirements.txt index bd549f9a947a4bc1fe44a6246da982b7ab3bda8c..93367ec8f4c23c5b645d837dd718b1774a3050b3 100644 GIT binary patch delta 157 zcmdnRaf5TiB_{nuhH?g5AT(#tW3T{XLk3<3E`|bzA_kC5GDAL)l?qm21e7(|e3VI- zQL}&{1t?ttl+Xn#$pF&H4B0@k2&~WmYz)XSgUPR%{f)|jGD!@@K%*e4O2E2u7&3w4 Rc|cP@)*Av bool: ... @abstractmethod - def fetch(self, app_id: str, metric: str) -> str: ... + def fetch(self, app_id: int, metric: str) -> str: ... diff --git a/source/adapters/itunes.py b/source/adapters/itunes.py index 44d1b34..4669995 100644 --- a/source/adapters/itunes.py +++ b/source/adapters/itunes.py @@ -16,7 +16,7 @@ class ItunesSearchAPIAdapter(SourceAdapter): def supports_metric(self, metric: str) -> bool: return metric in self.supported_metrics - def fetch(self, app_id: str, metric: str): + def fetch(self, app_id: int, metric: str): try: app = App.objects.get(id=app_id) except App.DoesNotExist: diff --git a/source/adapters/news.py b/source/adapters/news.py index 817c28c..727f270 100644 --- a/source/adapters/news.py +++ b/source/adapters/news.py @@ -22,7 +22,7 @@ def __init__(self, api_key=None): def supports_metric(self, metric: str) -> bool: return metric in self.supported_metrics - def fetch(self, app_id: str, metric: str): + def fetch(self, app_id: int, metric: str): try: app = App.objects.get(id=app_id) except App.DoesNotExist: diff --git a/source/adapters/reddit.py b/source/adapters/reddit.py new file mode 100644 index 0000000..c66a522 --- /dev/null +++ b/source/adapters/reddit.py @@ -0,0 +1,47 @@ +import os + +import praw + +from app.models import App +from metric.constants import MetricCodes +from source.adapters.base import SourceAdapter +from source.constants.source_type import SourceType + + +class RedditAPIAdapter(SourceAdapter): + code = "reddit" + name = "Reddit API" + type = SourceType.API + url = "https://oauth.reddit.com" + supported_metrics = [MetricCodes.DAILY_REDDIT_MENTIONS] + + def __init__(self): + self.reddit = praw.Reddit( + client_id=os.environ.get("REDDIT_CLIENT_ID"), + client_secret=os.environ.get("REDDIT_CLIENT_SECRET"), + user_agent=os.environ.get("REDDIT_USER_AGENT"), + ) + + def supports_metric(self, metric: str) -> bool: + return metric in self.supported_metrics + + def fetch(self, app_id: int, metric: str): + if metric != MetricCodes.DAILY_REDDIT_MENTIONS: + return None + + try: + app = App.objects.get(id=app_id) + except App.DoesNotExist: + return None + + query = app.name.lower() + + try: + results = self.reddit.subreddit("all").search(query, time_filter="day", sort="new") + except Exception as e: + print(f"[RedditSearchAPIAdapter] Error fetching data: {e}") + return None + + return { + MetricCodes.DAILY_REDDIT_MENTIONS: sum(1 for _ in results), + }.get(metric) diff --git a/source/services/source_registration_service.py b/source/services/source_registration_service.py index f77fa0d..63c1c08 100644 --- a/source/services/source_registration_service.py +++ b/source/services/source_registration_service.py @@ -4,6 +4,7 @@ # isort: off import source.adapters.itunes as _ # noqa import source.adapters.news as _ # noqa +import source.adapters.reddit as _ # noqa # isort: on From 0b8c5de5ca111ffe6da0e7812a9013b30931b6da Mon Sep 17 00:00:00 2001 From: Anyer Date: Fri, 18 Apr 2025 15:12:10 +0200 Subject: [PATCH 13/42] feat(test): Escriure tests per al connector de Reddit Refs #72 --- metric/admin.py | 8 ++++- metric/management/commands/poll_metrics.py | 5 +++- source/adapters/reddit.py | 23 ++++++++++++--- tests/unit/test_reddit_adapter.py | 34 ++++++++++++++++++++++ 4 files changed, 64 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_reddit_adapter.py diff --git a/metric/admin.py b/metric/admin.py index c7c8297..fceef0b 100644 --- a/metric/admin.py +++ b/metric/admin.py @@ -3,4 +3,10 @@ from .models import Metric, MetricValue admin.site.register(Metric) -admin.site.register(MetricValue) + + +@admin.register(MetricValue) +class MetricValueAdmin(admin.ModelAdmin): + list_display = ("app", "metric", "source", "value", "retrieved_at") + ordering = ("-retrieved_at", "-metric") + list_filter = ("metric", "source", "retrieved_at") diff --git a/metric/management/commands/poll_metrics.py b/metric/management/commands/poll_metrics.py index 55be67e..a6c1284 100644 --- a/metric/management/commands/poll_metrics.py +++ b/metric/management/commands/poll_metrics.py @@ -43,5 +43,8 @@ def handle(self, *args, **options): value.save() print(f" ✅ Valor guardat: {value.value} des de {value.source}") else: - print(f" ℹ️ Ja existia: {metric.code} per {app.code} via {value.source}") + print( + f" ℹ️ Ja existia: {metric.code} per {app.code} via {value.source}," + f" nou valor: {value.value}" + ) print("✅ Procés completat.") diff --git a/source/adapters/reddit.py b/source/adapters/reddit.py index c66a522..33849fc 100644 --- a/source/adapters/reddit.py +++ b/source/adapters/reddit.py @@ -35,13 +35,28 @@ def fetch(self, app_id: int, metric: str): return None query = app.name.lower() - + total = 0 + after = None try: - results = self.reddit.subreddit("all").search(query, time_filter="day", sort="new") + while True: + results = self.reddit.subreddit("all").search( + query, + time_filter="day", + sort="new", + limit=100, + params={"after": after} if after else {}, + ) + batch = list(results) + total += len(batch) + + if len(batch) < 100: + break + + after = batch[-1].fullname # exemple: 't3_abcdef' except Exception as e: - print(f"[RedditSearchAPIAdapter] Error fetching data: {e}") + print(f"[RedditAPIAdapter] Error fetching paginated data: {e}") return None return { - MetricCodes.DAILY_REDDIT_MENTIONS: sum(1 for _ in results), + MetricCodes.DAILY_REDDIT_MENTIONS: total, }.get(metric) diff --git a/tests/unit/test_reddit_adapter.py b/tests/unit/test_reddit_adapter.py new file mode 100644 index 0000000..dfd4753 --- /dev/null +++ b/tests/unit/test_reddit_adapter.py @@ -0,0 +1,34 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from app.models import App +from metric.constants import MetricCodes +from source.adapters.reddit import RedditAPIAdapter + + +@pytest.mark.django_db +@patch("source.adapters.reddit.praw.Reddit") +def test_reddit_api_adapter_fetch_mentions(mock_praw): + app = App.objects.create( + code="discord", + name="Discord", + appstore_id="985746746", + playstore_id="com.discord", + description="App de comunicació global", + ) + + # Mock del resultat de la crida a Reddit + mock_instance = MagicMock() + mock_results = [1, 2, 3] # Simulem 3 resultats trobats + mock_instance.subreddit.return_value.search.return_value = mock_results + mock_praw.return_value = mock_instance + + adapter = RedditAPIAdapter() + + # Comprovació de supports_metric + assert adapter.supports_metric(MetricCodes.DAILY_REDDIT_MENTIONS) is True + + # Comprovació del fetch (amb resultats simulats) + count = adapter.fetch(app.id, MetricCodes.DAILY_REDDIT_MENTIONS) + assert count == 3 From a9049d5b93edaa3f948d320734e6006ecf9f90e4 Mon Sep 17 00:00:00 2001 From: Anyer Date: Fri, 18 Apr 2025 18:12:44 +0200 Subject: [PATCH 14/42] =?UTF-8?q?feat(google-play):=20Crear=20el=20connect?= =?UTF-8?q?or=20de=20Google=20Play=20dins=20l=E2=80=99app=20sources?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #57 --- requirements.txt | Bin 1240 -> 1296 bytes source/adapters/base.py | 2 +- source/adapters/google_play_scraper.py | 39 +++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 source/adapters/google_play_scraper.py diff --git a/requirements.txt b/requirements.txt index 93367ec8f4c23c5b645d837dd718b1774a3050b3..e42532c167713b71117e585be37596401253781a 100644 GIT binary patch delta 64 zcmcb?Ie}|K8KZGJLp~6uGvqL&GUzfC0O>@AN+7G4A(^2F$Swei7ctlZp&^4FgAov$ KZ!Tm^X9fV_NDWc| delta 12 TcmbQhb%S$58RO=4j7iJ@AG`!= diff --git a/source/adapters/base.py b/source/adapters/base.py index 316f350..ca4a01e 100644 --- a/source/adapters/base.py +++ b/source/adapters/base.py @@ -16,4 +16,4 @@ class SourceAdapter(ABC): def supports_metric(self, metric: str) -> bool: ... @abstractmethod - def fetch(self, app_id: int, metric: str) -> str: ... + def fetch(self, app_id: int, metric: str) -> str | None: ... diff --git a/source/adapters/google_play_scraper.py b/source/adapters/google_play_scraper.py new file mode 100644 index 0000000..7b6dfe6 --- /dev/null +++ b/source/adapters/google_play_scraper.py @@ -0,0 +1,39 @@ +from google_play_scraper import app as gp_app + +from app.models import App +from metric.constants import MetricCodes +from source.adapters.base import SourceAdapter +from source.constants.source_type import SourceType + + +class GooglePlayScraperAdapter(SourceAdapter): + code = "google_play" + name = "Google Play Scraper" + type = SourceType.SCRAPER + url = "https://play.google.com" + supported_metrics = [MetricCodes.AVERAGE_RATING, MetricCodes.TOTAL_REVIEWS] + + def supports_metric(self, metric: str): + return metric in self.supported_metrics + + def fetch(self, app_id: int, metric: str): + try: + app_instance = App.objects.get(id=app_id) + package_name = app_instance.playstore_id + except App.DoesNotExist: + print(f"[GooglePlayScraperAdapter] App with id {app_id} does not exist.") + return None + except AttributeError: + print("[GooglePlayScraperAdapter] App is missing 'playstore_id' field.") + return None + + try: + result = gp_app(package_name, lang="en", country="us") + except Exception as e: + print(f"[GooglePlayScraperAdapter] Error fetching data: {e}") + return None + print(result) + return { + MetricCodes.AVERAGE_RATING: result.get("score"), + MetricCodes.TOTAL_REVIEWS: result.get("reviews"), + }.get(metric) From d997c52f2fb8bce2a4201b346c6f2f764724d585 Mon Sep 17 00:00:00 2001 From: Anyer Date: Fri, 18 Apr 2025 18:14:04 +0200 Subject: [PATCH 15/42] feat(test-google-play): Escriure tests per als connectors de Google Play Refs #114 --- .../unit/test_google_play_scraper_adapter.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 tests/unit/test_google_play_scraper_adapter.py diff --git a/tests/unit/test_google_play_scraper_adapter.py b/tests/unit/test_google_play_scraper_adapter.py new file mode 100644 index 0000000..8c29529 --- /dev/null +++ b/tests/unit/test_google_play_scraper_adapter.py @@ -0,0 +1,27 @@ +from unittest.mock import patch + +import pytest + +from app.models import App +from metric.constants import MetricCodes +from source.adapters.google_play_scraper import GooglePlayScraperAdapter + + +@pytest.mark.django_db +@patch("source.adapters.google_play_scraper.gp_app") +def test_google_play_scraper_adapter_fetch(mock_gp_app): + app = App.objects.create(name="Discord", playstore_id="com.discord") + + # Simulem la resposta de google_play_scraper.app() + mock_gp_app.return_value = {"score": 4.6, "reviews": 7890} + + adapter = GooglePlayScraperAdapter() + + assert adapter.supports_metric(MetricCodes.AVERAGE_RATING) + assert adapter.supports_metric(MetricCodes.TOTAL_REVIEWS) + + rating = adapter.fetch(app.id, MetricCodes.AVERAGE_RATING) + reviews = adapter.fetch(app.id, MetricCodes.TOTAL_REVIEWS) + + assert rating == 4.6 + assert reviews == 7890 From 783b1d767b86d6eed886edee52272efc053e0cfc Mon Sep 17 00:00:00 2001 From: Anyer Date: Fri, 18 Apr 2025 19:03:26 +0200 Subject: [PATCH 16/42] =?UTF-8?q?feat(google-play):=20Implementar=20la=20f?= =?UTF-8?q?unci=C3=B3=20fetch=20per=20obtenir=20dades=20d=E2=80=99una=20ap?= =?UTF-8?q?p=20concreta?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #58 --- metric/constants/__init__.py | 4 +- metric/management/commands/poll_metrics.py | 100 ++++++++++++------ source/adapters/google_play_scraper.py | 11 +- source/adapters/reddit.py | 6 +- .../services/source_registration_service.py | 1 + 5 files changed, 81 insertions(+), 41 deletions(-) diff --git a/metric/constants/__init__.py b/metric/constants/__init__.py index 582ff47..232a183 100644 --- a/metric/constants/__init__.py +++ b/metric/constants/__init__.py @@ -2,4 +2,6 @@ class MetricCodes: AVERAGE_RATING = "average_rating" TOTAL_REVIEWS = "total_reviews" DAILY_NEWS_BLOG_MENTIONS = "daily_news_blog_mentions" - DAILY_REDDIT_MENTIONS = "daily_reddit_mentions" + DAILY_SOCIAL_NETWORK_MENTIONS = "daily_social_network_mentions" + TOTAL_DOWNLOADS = "total_downloads" + LAST_UPDATE_DATE = "last_update_date" diff --git a/metric/management/commands/poll_metrics.py b/metric/management/commands/poll_metrics.py index a6c1284..5d5e015 100644 --- a/metric/management/commands/poll_metrics.py +++ b/metric/management/commands/poll_metrics.py @@ -2,49 +2,79 @@ from django.utils import timezone from app.models import App +from metric.constants import MetricCodes from metric.models import Metric from metric.strategies.generic import GenericMetricStrategy from source.services.source_registration_service import SourceRegistrationService +def validate_metric_codes(): + # ✅ Validació entre MetricCodes i Metric a BD + codes_in_db = set(Metric.objects.values_list("code", flat=True)) + codes_in_code = { + v for k, v in MetricCodes.__dict__.items() if not k.startswith("__") and not callable(v) + } + + extra_in_db = codes_in_db - codes_in_code + missing_in_db = codes_in_code - codes_in_db + + if extra_in_db: + print("❌ Les següents mètriques estan a la BD però no a MetricCodes:") + for code in sorted(extra_in_db): + print(f" - {code}") + + if missing_in_db: + print("❌ Les següents MetricCodes no estan registrades a la BD:") + for code in sorted(missing_in_db): + print(f" - {code}") + + if not extra_in_db and not missing_in_db: + print("✅ Coherència correcta entre MetricCodes i BD.") + return True + else: + print("❌ Hi ha inconsistències entre MetricCodes i BD.") + return False + + class Command(BaseCommand): help = "Recull mètriques per a totes les apps i fonts disponibles" def handle(self, *args, **options): print("[🔁] Iniciant recollida de mètriques diàries...") - adapters = SourceRegistrationService().load_and_register() - apps = App.objects.all() - metrics = Metric.objects.prefetch_related("sources").all() - - for metric in metrics: - print(f"📌 Processant mètrica: {metric.code}") - strategy = GenericMetricStrategy(metric.code) - source_codes = metric.sources.values_list("code", flat=True) - matching_adapters = [a for a in adapters if a.code in source_codes] - - for app in apps: - print(f" 🔍 App: {app.code}") - values = strategy.compute_all(app.id, matching_adapters) - - for value in values: - if value.value is None: - print(f" ⚠️ {metric.code} ({value.source}) no ha retornat valor.") - continue - - exists = value.__class__.objects.filter( - app=app, - metric=metric, - source=value.source, - retrieved_at__date=timezone.now().date(), - ).exists() - - if not exists: - value.save() - print(f" ✅ Valor guardat: {value.value} des de {value.source}") - else: - print( - f" ℹ️ Ja existia: {metric.code} per {app.code} via {value.source}," - f" nou valor: {value.value}" - ) - print("✅ Procés completat.") + if validate_metric_codes(): + adapters = SourceRegistrationService().load_and_register() + apps = App.objects.all() + metrics = Metric.objects.prefetch_related("sources").all() + + for metric in metrics: + print(f"📌 Processant mètrica: {metric.code}") + strategy = GenericMetricStrategy(metric.code) + source_codes = metric.sources.values_list("code", flat=True) + matching_adapters = [a for a in adapters if a.code in source_codes] + + for app in apps: + print(f" 🔍 App: {app.code}") + values = strategy.compute_all(app.id, matching_adapters) + + for value in values: + if value.value is None: + print(f" ⚠️ {metric.code} ({value.source}) no ha retornat valor.") + continue + + exists = value.__class__.objects.filter( + app=app, + metric=metric, + source=value.source, + retrieved_at__date=timezone.now().date(), + ).exists() + + if not exists: + value.save() + print(f" ✅ Valor guardat: {value.value} des de {value.source}") + else: + print( + f" ℹ️ Ja existia: {metric.code} per {app.code}" + f" via {value.source}, nou valor: {value.value}" + ) + print("✅ Procés completat.") diff --git a/source/adapters/google_play_scraper.py b/source/adapters/google_play_scraper.py index 7b6dfe6..2cf96fc 100644 --- a/source/adapters/google_play_scraper.py +++ b/source/adapters/google_play_scraper.py @@ -11,7 +11,12 @@ class GooglePlayScraperAdapter(SourceAdapter): name = "Google Play Scraper" type = SourceType.SCRAPER url = "https://play.google.com" - supported_metrics = [MetricCodes.AVERAGE_RATING, MetricCodes.TOTAL_REVIEWS] + supported_metrics = [ + MetricCodes.AVERAGE_RATING, + MetricCodes.TOTAL_REVIEWS, + MetricCodes.TOTAL_DOWNLOADS, + MetricCodes.LAST_UPDATE_DATE, + ] def supports_metric(self, metric: str): return metric in self.supported_metrics @@ -32,8 +37,10 @@ def fetch(self, app_id: int, metric: str): except Exception as e: print(f"[GooglePlayScraperAdapter] Error fetching data: {e}") return None - print(result) + return { MetricCodes.AVERAGE_RATING: result.get("score"), MetricCodes.TOTAL_REVIEWS: result.get("reviews"), + MetricCodes.TOTAL_DOWNLOADS: result.get("realInstalls"), + MetricCodes.LAST_UPDATE_DATE: result.get("lastUpdatedOn"), }.get(metric) diff --git a/source/adapters/reddit.py b/source/adapters/reddit.py index 33849fc..1de618f 100644 --- a/source/adapters/reddit.py +++ b/source/adapters/reddit.py @@ -13,7 +13,7 @@ class RedditAPIAdapter(SourceAdapter): name = "Reddit API" type = SourceType.API url = "https://oauth.reddit.com" - supported_metrics = [MetricCodes.DAILY_REDDIT_MENTIONS] + supported_metrics = [MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS] def __init__(self): self.reddit = praw.Reddit( @@ -26,7 +26,7 @@ def supports_metric(self, metric: str) -> bool: return metric in self.supported_metrics def fetch(self, app_id: int, metric: str): - if metric != MetricCodes.DAILY_REDDIT_MENTIONS: + if metric != MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS: return None try: @@ -58,5 +58,5 @@ def fetch(self, app_id: int, metric: str): return None return { - MetricCodes.DAILY_REDDIT_MENTIONS: total, + MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS: total, }.get(metric) diff --git a/source/services/source_registration_service.py b/source/services/source_registration_service.py index 63c1c08..c3bf86b 100644 --- a/source/services/source_registration_service.py +++ b/source/services/source_registration_service.py @@ -5,6 +5,7 @@ import source.adapters.itunes as _ # noqa import source.adapters.news as _ # noqa import source.adapters.reddit as _ # noqa +import source.adapters.google_play_scraper as _ # noqa # isort: on From f2b6518e1eae8c7e8fc6faed36a561e11af34c3e Mon Sep 17 00:00:00 2001 From: Anyer Date: Fri, 25 Apr 2025 16:45:09 +0200 Subject: [PATCH 17/42] =?UTF-8?q?feat(google-play):=20Implementar=20la=20f?= =?UTF-8?q?unci=C3=B3=20per=20a=20obtenir=20les=20reviews=20de=20google=20?= =?UTF-8?q?play?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #56 --- metric/management/commands/poll_reviews.py | 17 +++++++++++++++ source/adapters/google_play_scraper.py | 25 ++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 metric/management/commands/poll_reviews.py diff --git a/metric/management/commands/poll_reviews.py b/metric/management/commands/poll_reviews.py new file mode 100644 index 0000000..9e244c2 --- /dev/null +++ b/metric/management/commands/poll_reviews.py @@ -0,0 +1,17 @@ +from django.core.management.base import BaseCommand + +from app.models import App +from source.adapters.google_play_scraper import GooglePlayScraperAdapter + + +class Command(BaseCommand): + help = "Recull las resseynes del dia d'ahir de google play" + + def handle(self, *args, **options): + print("[🔁] Iniciant recollida de ressenyes diàries...") + + apps = App.objects.all() + for app in apps: + GooglePlayScraperAdapter().fetch_reviews(app.id) + + print("✅ Procés completat.") diff --git a/source/adapters/google_play_scraper.py b/source/adapters/google_play_scraper.py index 2cf96fc..d3c44e7 100644 --- a/source/adapters/google_play_scraper.py +++ b/source/adapters/google_play_scraper.py @@ -1,4 +1,8 @@ +from datetime import timedelta + +from django.utils import timezone from google_play_scraper import app as gp_app +from google_play_scraper import reviews from app.models import App from metric.constants import MetricCodes @@ -44,3 +48,24 @@ def fetch(self, app_id: int, metric: str): MetricCodes.TOTAL_DOWNLOADS: result.get("realInstalls"), MetricCodes.LAST_UPDATE_DATE: result.get("lastUpdatedOn"), }.get(metric) + + def fetch_reviews(self, app_id: int): + try: + app_instance = App.objects.get(id=app_id) + package_name = app_instance.playstore_id + except App.DoesNotExist: + print(f"[GooglePlayScraperAdapter] App with id {app_id} does not exist.") + return None + except AttributeError: + print("[GooglePlayScraperAdapter] App is missing 'playstore_id' field.") + return None + + yesterday = timezone.now().date() - timedelta(days=1) + + result, _ = reviews(package_name, count=200) + + reviews_from_yesterday = [r for r in result if r["at"].date() == yesterday] + + print(f"Ressenyes d'ahir: {reviews_from_yesterday}") + print(f"Ressenyes d'ahir: {len(reviews_from_yesterday)}") + return reviews_from_yesterday From d755ec6c1739025ac3df55d2155d06c1f694e48e Mon Sep 17 00:00:00 2001 From: Anyer Date: Fri, 25 Apr 2025 16:51:35 +0200 Subject: [PATCH 18/42] hotfix(api-test): Canvi de nom de DAILY_REDDIT_MENTIONS a DAILY_SOCIAL_NETWORK_MENTIONS --- metric/definitions/metrics.py | 2 +- tests/unit/test_reddit_adapter.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metric/definitions/metrics.py b/metric/definitions/metrics.py index 855f06d..a111d9f 100644 --- a/metric/definitions/metrics.py +++ b/metric/definitions/metrics.py @@ -21,7 +21,7 @@ "value_type": MetricValueType.INTEGER, }, { - "code": MetricCodes.DAILY_REDDIT_MENTIONS, + "code": MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS, "name": "Daily Reddit Mentions", "description": "Nombre de mencions diàries a Reddit", "value_type": MetricValueType.INTEGER, diff --git a/tests/unit/test_reddit_adapter.py b/tests/unit/test_reddit_adapter.py index dfd4753..60874ad 100644 --- a/tests/unit/test_reddit_adapter.py +++ b/tests/unit/test_reddit_adapter.py @@ -27,8 +27,8 @@ def test_reddit_api_adapter_fetch_mentions(mock_praw): adapter = RedditAPIAdapter() # Comprovació de supports_metric - assert adapter.supports_metric(MetricCodes.DAILY_REDDIT_MENTIONS) is True + assert adapter.supports_metric(MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS) is True # Comprovació del fetch (amb resultats simulats) - count = adapter.fetch(app.id, MetricCodes.DAILY_REDDIT_MENTIONS) + count = adapter.fetch(app.id, MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS) assert count == 3 From 3af6819ca3dea4667e73d95a9a8cd241e913eefa Mon Sep 17 00:00:00 2001 From: Anyer Date: Fri, 25 Apr 2025 16:57:44 +0200 Subject: [PATCH 19/42] feature(api-rest): Integrar DRF Spectacular al projecte (llibreries i configuracions) Refs #63 --- config/settings.py | 5 +++++ config/urls.py | 3 +++ requirements.txt | Bin 1296 -> 1756 bytes 3 files changed, 8 insertions(+) diff --git a/config/settings.py b/config/settings.py index 0da88a5..16eab80 100644 --- a/config/settings.py +++ b/config/settings.py @@ -49,6 +49,7 @@ "source", "review", "metric", + "drf_spectacular", ] MIDDLEWARE = [ @@ -133,3 +134,7 @@ # https://docs.djangoproject.com/en/5.1/ref/settings/#default-auto-field DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + +REST_FRAMEWORK = { + "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", +} diff --git a/config/urls.py b/config/urls.py index a7369d4..931428a 100644 --- a/config/urls.py +++ b/config/urls.py @@ -18,6 +18,7 @@ from django.contrib import admin from django.http import HttpResponse from django.urls import path +from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView def home(request): @@ -27,4 +28,6 @@ def home(request): urlpatterns = [ path("admin/", admin.site.urls), path("", home), + path("api/schema/", SpectacularAPIView.as_view(), name="schema"), + path("api/docs/", SpectacularSwaggerView.as_view(url_name="schema"), name="swagger-ui"), ] diff --git a/requirements.txt b/requirements.txt index e42532c167713b71117e585be37596401253781a..7c937320637ceac7443cdc35ec5d98210c9de846 100644 GIT binary patch delta 423 zcmZvYK~BOz6o&tSt{@zsi-ts3E~=dti5tSoB?oY28lW;Ll@yF&$q68t19%3*b=+|V z_x|ryaAPLzO#A--&G)~3>#MWB2y?8tDy%TzN^y&8WPFm(C^Xp|IZ8a?m5M1fYZN@? zSn%sz|30fv92R}?`5SW?PpB#o)3Ky-Okd9T8(z3q>PVyPH%&Dl*VD|VVuyI?P2?rA4wMh7kYr~4yuC0Z6j<8s@3tPei%G-9U&yhQU@dF LQy{Cccfs}tQy)$R delta 43 zcmV+`0M!574Uh_uBC$Xe0h5ja9FyDunv Date: Fri, 25 Apr 2025 19:04:57 +0200 Subject: [PATCH 20/42] =?UTF-8?q?feature(api-rest):=20Crear=20les=20views?= =?UTF-8?q?=20i=20urls=20de=20l=E2=80=99app=20app=20i=20documentar-ho?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #66 --- app/repositories.py | 23 ++++++++++++++++++++ app/serializers.py | 22 +++++++++++++++++++ app/services.py | 21 ++++++++++++++++++ app/urls.py | 11 ++++++++++ app/views.py | 52 +++++++++++++++++++++++++++++++++++++++++++++ config/urls.py | 3 ++- 6 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 app/repositories.py create mode 100644 app/serializers.py create mode 100644 app/services.py create mode 100644 app/urls.py diff --git a/app/repositories.py b/app/repositories.py new file mode 100644 index 0000000..09897cd --- /dev/null +++ b/app/repositories.py @@ -0,0 +1,23 @@ +from django.shortcuts import get_object_or_404 + +from .models import App + + +class AppRepository: + def get_all(self): + return App.objects.all() + + def get_by_id(self, app_id): + return get_object_or_404(App, id=app_id) + + def create(self, data): + return App.objects.create(**data) + + def update(self, instance, data): + for attr, value in data.items(): + setattr(instance, attr, value) + instance.save() + return instance + + def delete(self, instance): + instance.delete() diff --git a/app/serializers.py b/app/serializers.py new file mode 100644 index 0000000..453bc00 --- /dev/null +++ b/app/serializers.py @@ -0,0 +1,22 @@ +from rest_framework import serializers + +from .models import App + + +class AppSerializer(serializers.ModelSerializer): + class Meta: + model = App + fields = [ + "id", + "code", + "name", + "description", + "appstore_id", + "playstore_id", + "developer", + "available_on_ios", + "available_on_android", + "pegi_rating", + "release_date", + "min_ios_version", + ] diff --git a/app/services.py b/app/services.py new file mode 100644 index 0000000..6afd622 --- /dev/null +++ b/app/services.py @@ -0,0 +1,21 @@ +from .repositories import AppRepository + + +class AppService: + def __init__(self): + self.repo = AppRepository() + + def list_apps(self): + return self.repo.get_all() + + def get_app(self, app_id): + return self.repo.get_by_id(app_id) + + def create_app(self, validated_data): + return self.repo.create(validated_data) + + def update_app(self, instance, validated_data): + return self.repo.update(instance, validated_data) + + def delete_app(self, instance): + return self.repo.delete(instance) diff --git a/app/urls.py b/app/urls.py new file mode 100644 index 0000000..8118486 --- /dev/null +++ b/app/urls.py @@ -0,0 +1,11 @@ +from django.urls import include, path +from rest_framework.routers import DefaultRouter + +from .views import AppViewSet + +router = DefaultRouter() +router.register(r"apps", AppViewSet, basename="app") + +urlpatterns = [ + path("", include(router.urls)), +] diff --git a/app/views.py b/app/views.py index e69de29..cb25bfa 100644 --- a/app/views.py +++ b/app/views.py @@ -0,0 +1,52 @@ +from drf_spectacular.utils import OpenApiParameter, extend_schema +from rest_framework import status, viewsets +from rest_framework.response import Response + +from .repositories import AppRepository +from .serializers import AppSerializer +from .services import AppService + + +class AppViewSet(viewsets.ViewSet): + service = AppService() + repo = AppRepository() + + @extend_schema(responses=AppSerializer(many=True)) + def list(self, request): + apps = self.service.list_apps() + serializer = AppSerializer(apps, many=True) + return Response(serializer.data) + + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses=AppSerializer, + ) + def retrieve(self, request, pk=None): + app = self.service.get_app(pk) + serializer = AppSerializer(app) + return Response(serializer.data) + + @extend_schema(request=AppSerializer, responses=AppSerializer) + def create(self, request): + print("REQUEST DATA:", request.data) + serializer = AppSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + app = self.service.create_app(serializer.validated_data) + return Response(AppSerializer(app).data, status=status.HTTP_201_CREATED) + + @extend_schema(request=AppSerializer, responses=AppSerializer) + def update(self, request, pk=None): + app = self.service.get_app(pk) + serializer = AppSerializer(app, data=request.data) + serializer.is_valid(raise_exception=True) + updated_app = self.service.update_app(app, serializer.validated_data) + return Response(AppSerializer(updated_app).data) + + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses={204: None}, + ) + def destroy(self, request, pk=None): + app = self.service.get_app(pk) + self.service.delete_app(app) + return Response(status=status.HTTP_204_NO_CONTENT) diff --git a/config/urls.py b/config/urls.py index 931428a..4794422 100644 --- a/config/urls.py +++ b/config/urls.py @@ -17,7 +17,7 @@ from django.contrib import admin from django.http import HttpResponse -from django.urls import path +from django.urls import include, path from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView @@ -30,4 +30,5 @@ def home(request): path("", home), path("api/schema/", SpectacularAPIView.as_view(), name="schema"), path("api/docs/", SpectacularSwaggerView.as_view(url_name="schema"), name="swagger-ui"), + path("api/", include("app.urls")), ] From e81fc7bad131d1238c8d85444dd89a0b1f1cf42e Mon Sep 17 00:00:00 2001 From: Anyer Date: Sun, 27 Apr 2025 21:39:24 +0200 Subject: [PATCH 21/42] =?UTF-8?q?feature(api-rest):=20Crear=20les=20views?= =?UTF-8?q?=20i=20urls=20de=20l=E2=80=99app=20app=20i=20documentar-ho?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #66 --- app/migrations/0003_alter_app_developer.py | 18 +++++++ app/models.py | 2 +- app/serializers.py | 6 +++ app/services.py | 55 +++++++++++++++++++++- app/views.py | 17 +++++-- source/adapters/google_play_scraper.py | 15 ++++-- source/adapters/itunes.py | 18 ++++--- 7 files changed, 115 insertions(+), 16 deletions(-) create mode 100644 app/migrations/0003_alter_app_developer.py diff --git a/app/migrations/0003_alter_app_developer.py b/app/migrations/0003_alter_app_developer.py new file mode 100644 index 0000000..099d203 --- /dev/null +++ b/app/migrations/0003_alter_app_developer.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.7 on 2025-04-27 12:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("app", "0002_app_code"), + ] + + operations = [ + migrations.AlterField( + model_name="app", + name="developer", + field=models.CharField(max_length=100, null=True), + ), + ] diff --git a/app/models.py b/app/models.py index 1e835ab..bf7c3bf 100644 --- a/app/models.py +++ b/app/models.py @@ -7,7 +7,7 @@ class App(models.Model): description = models.TextField() appstore_id = models.CharField(max_length=100, null=True, unique=True) playstore_id = models.CharField(max_length=100, null=True, unique=True) - developer = models.CharField(max_length=100, null=True, unique=True) + developer = models.CharField(max_length=100, null=True) available_on_ios = models.BooleanField(default=False) available_on_android = models.BooleanField(default=False) pegi_rating = models.CharField(max_length=10, null=True) # PEGI rating diff --git a/app/serializers.py b/app/serializers.py index 453bc00..c8834aa 100644 --- a/app/serializers.py +++ b/app/serializers.py @@ -20,3 +20,9 @@ class Meta: "release_date", "min_ios_version", ] + + +class AppCreateSerializer(serializers.ModelSerializer): + class Meta: + model = App + fields = ["code", "name", "description", "appstore_id", "playstore_id"] diff --git a/app/services.py b/app/services.py index 6afd622..47b4fc9 100644 --- a/app/services.py +++ b/app/services.py @@ -1,9 +1,16 @@ +from datetime import datetime + +from source.adapters.google_play_scraper import GooglePlayScraperAdapter +from source.adapters.itunes import ItunesSearchAPIAdapter + from .repositories import AppRepository class AppService: - def __init__(self): + def __init__(self, itunes_adapter=None, google_play_adapter=None): self.repo = AppRepository() + self.itunes_adapter = itunes_adapter or ItunesSearchAPIAdapter() + self.google_play_adapter = google_play_adapter or GooglePlayScraperAdapter() def list_apps(self): return self.repo.get_all() @@ -12,8 +19,54 @@ def get_app(self, app_id): return self.repo.get_by_id(app_id) def create_app(self, validated_data): + self._fetch_appstore_data(validated_data) + self._fetch_playstore_data(validated_data) return self.repo.create(validated_data) + def _fetch_appstore_data(self, validated_data): + if not validated_data.get("appstore_id"): + return + + result = self.itunes_adapter.lookup_app(validated_data["appstore_id"]) + if not result: + return + + validated_data["available_on_ios"] = True + validated_data["developer"] = result.get("artistName") + validated_data["pegi_rating"] = result.get("contentAdvisoryRating") + validated_data["min_ios_version"] = result.get("minimumOsVersion") + + released_str = result.get("releaseDate") + if released_str: + try: + if "T" in released_str: + release_date = datetime.fromisoformat(released_str.replace("Z", "")).date() + validated_data["release_date"] = release_date + except Exception as e: + print(f"Error parsejant release_date (AppStore): {released_str} ({e})") + + def _fetch_playstore_data(self, validated_data): + if not validated_data.get("playstore_id"): + return + + result = self.google_play_adapter.lookup_app(validated_data["playstore_id"]) + if not result: + return + + validated_data["available_on_android"] = True + validated_data["developer"] = validated_data.get("developer") or result.get("developer") + validated_data["pegi_rating"] = validated_data.get("pegi_rating") or result.get( + "contentRating" + ) + + released_str = result.get("released") + if released_str and not validated_data.get("release_date"): + try: + release_date = datetime.strptime(released_str, "%b %d, %Y").date() + validated_data["release_date"] = release_date + except ValueError: + print(f"Format de data invàlid (PlayStore): {released_str}") + def update_app(self, instance, validated_data): return self.repo.update(instance, validated_data) diff --git a/app/views.py b/app/views.py index cb25bfa..7662072 100644 --- a/app/views.py +++ b/app/views.py @@ -3,7 +3,7 @@ from rest_framework.response import Response from .repositories import AppRepository -from .serializers import AppSerializer +from .serializers import AppCreateSerializer, AppSerializer from .services import AppService @@ -26,10 +26,21 @@ def retrieve(self, request, pk=None): serializer = AppSerializer(app) return Response(serializer.data) - @extend_schema(request=AppSerializer, responses=AppSerializer) + @extend_schema( + request=AppCreateSerializer, + responses=AppSerializer, + description=( + "Creates a new app.\n\n" + "The fields 'available_on_ios', 'available_on_android', 'pegi_rating', " + "'release_date', and 'min_ios_version' will be automatically populated " + "by fetching data from the App Store and/or Google Play APIs based " + "on the provided identifiers.\n\n" + "You do not need to manually provide these fields when creating an app." + ), + ) def create(self, request): print("REQUEST DATA:", request.data) - serializer = AppSerializer(data=request.data) + serializer = AppCreateSerializer(data=request.data) serializer.is_valid(raise_exception=True) app = self.service.create_app(serializer.validated_data) return Response(AppSerializer(app).data, status=status.HTTP_201_CREATED) diff --git a/source/adapters/google_play_scraper.py b/source/adapters/google_play_scraper.py index d3c44e7..d92c994 100644 --- a/source/adapters/google_play_scraper.py +++ b/source/adapters/google_play_scraper.py @@ -36,11 +36,7 @@ def fetch(self, app_id: int, metric: str): print("[GooglePlayScraperAdapter] App is missing 'playstore_id' field.") return None - try: - result = gp_app(package_name, lang="en", country="us") - except Exception as e: - print(f"[GooglePlayScraperAdapter] Error fetching data: {e}") - return None + result = self.lookup_app(package_name) return { MetricCodes.AVERAGE_RATING: result.get("score"), @@ -49,6 +45,15 @@ def fetch(self, app_id: int, metric: str): MetricCodes.LAST_UPDATE_DATE: result.get("lastUpdatedOn"), }.get(metric) + @staticmethod + def lookup_app(package_name: str) -> dict | None: + try: + result = gp_app(package_name, lang="en", country="uk") + return result + except Exception as e: + print(f"[GooglePlayScraperAdapter] Error fetching app data: {e}") + return None + def fetch_reviews(self, app_id: int): try: app_instance = App.objects.get(id=app_id) diff --git a/source/adapters/itunes.py b/source/adapters/itunes.py index 4669995..493958e 100644 --- a/source/adapters/itunes.py +++ b/source/adapters/itunes.py @@ -22,7 +22,17 @@ def fetch(self, app_id: int, metric: str): except App.DoesNotExist: return None - response = requests.get(f"{self.url}/lookup?id={app.appstore_id}") + result = self.lookup_app(app.appstore_id) + if not result: + return None + + return { + MetricCodes.AVERAGE_RATING: result.get("averageUserRating"), + MetricCodes.TOTAL_REVIEWS: result.get("userRatingCount"), + }.get(metric) + + def lookup_app(self, appstore_id: str) -> dict | None: + response = requests.get(f"{self.url}/lookup?id={appstore_id}") if not response.ok: return None @@ -32,8 +42,4 @@ def fetch(self, app_id: int, metric: str): if not results: return None - result = results[0] - return { - MetricCodes.AVERAGE_RATING: result.get("averageUserRating"), - MetricCodes.TOTAL_REVIEWS: result.get("userRatingCount"), - }.get(metric) + return results[0] From 7de3abd4a20066eed12b6ea99e4b9db99e1b2926 Mon Sep 17 00:00:00 2001 From: Anyer Date: Mon, 28 Apr 2025 22:21:01 +0200 Subject: [PATCH 22/42] =?UTF-8?q?feature(api-rest):=20Crear=20les=20views?= =?UTF-8?q?=20i=20urls=20de=20l=E2=80=99app=20source=20i=20documentar-ho?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #65 --- app/views.py | 2 - config/urls.py | 1 + metric/management/commands/poll_metrics.py | 4 +- metric/serializers.py | 7 ++ source/repositories.py | 23 ++++ source/repositories/__init__.py | 0 source/repositories/source_repository.py | 7 -- source/serializers.py | 28 +++++ source/services.py | 35 ++++++ source/services/__init__.py | 0 .../services/source_registration_service.py | 32 ----- source/urls.py | 11 ++ source/views.py | 112 ++++++++++++++++++ tests/integration/test_end_to_end.py | 4 +- .../unit/test_source_registration_service.py | 10 +- 15 files changed, 227 insertions(+), 49 deletions(-) create mode 100644 metric/serializers.py create mode 100644 source/repositories.py delete mode 100644 source/repositories/__init__.py delete mode 100644 source/repositories/source_repository.py create mode 100644 source/serializers.py create mode 100644 source/services.py delete mode 100644 source/services/__init__.py delete mode 100644 source/services/source_registration_service.py create mode 100644 source/urls.py diff --git a/app/views.py b/app/views.py index 7662072..748f184 100644 --- a/app/views.py +++ b/app/views.py @@ -2,14 +2,12 @@ from rest_framework import status, viewsets from rest_framework.response import Response -from .repositories import AppRepository from .serializers import AppCreateSerializer, AppSerializer from .services import AppService class AppViewSet(viewsets.ViewSet): service = AppService() - repo = AppRepository() @extend_schema(responses=AppSerializer(many=True)) def list(self, request): diff --git a/config/urls.py b/config/urls.py index 4794422..704e0a6 100644 --- a/config/urls.py +++ b/config/urls.py @@ -31,4 +31,5 @@ def home(request): path("api/schema/", SpectacularAPIView.as_view(), name="schema"), path("api/docs/", SpectacularSwaggerView.as_view(url_name="schema"), name="swagger-ui"), path("api/", include("app.urls")), + path("api/", include("source.urls")), ] diff --git a/metric/management/commands/poll_metrics.py b/metric/management/commands/poll_metrics.py index 5d5e015..6a9e943 100644 --- a/metric/management/commands/poll_metrics.py +++ b/metric/management/commands/poll_metrics.py @@ -5,7 +5,7 @@ from metric.constants import MetricCodes from metric.models import Metric from metric.strategies.generic import GenericMetricStrategy -from source.services.source_registration_service import SourceRegistrationService +from source.services import SourceService def validate_metric_codes(): @@ -43,7 +43,7 @@ def handle(self, *args, **options): print("[🔁] Iniciant recollida de mètriques diàries...") if validate_metric_codes(): - adapters = SourceRegistrationService().load_and_register() + adapters = SourceService().load_sources() apps = App.objects.all() metrics = Metric.objects.prefetch_related("sources").all() diff --git a/metric/serializers.py b/metric/serializers.py new file mode 100644 index 0000000..e07a3b5 --- /dev/null +++ b/metric/serializers.py @@ -0,0 +1,7 @@ +from rest_framework import serializers + + +class LinkMetricsSerializer(serializers.Serializer): + metrics = serializers.ListField( + child=serializers.IntegerField(), help_text="List of metric IDs to link to the source." + ) diff --git a/source/repositories.py b/source/repositories.py new file mode 100644 index 0000000..7f5b9f1 --- /dev/null +++ b/source/repositories.py @@ -0,0 +1,23 @@ +from django.shortcuts import get_object_or_404 + +from .models import Source + + +class SourceRepository: + def get_all(self): + return Source.objects.all() + + def get_by_id(self, app_id): + return get_object_or_404(Source, id=app_id) + + def create(self, data): + return Source.objects.create(**data) + + def update(self, instance, data): + for attr, value in data.items(): + setattr(instance, attr, value) + instance.save() + return instance + + def delete(self, instance): + instance.delete() diff --git a/source/repositories/__init__.py b/source/repositories/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/source/repositories/source_repository.py b/source/repositories/source_repository.py deleted file mode 100644 index 7ae5108..0000000 --- a/source/repositories/source_repository.py +++ /dev/null @@ -1,7 +0,0 @@ -from source.models import Source - - -class SourceRepository: - @staticmethod - def get_or_create(code, defaults): - return Source.objects.get_or_create(code=code, defaults=defaults) diff --git a/source/serializers.py b/source/serializers.py new file mode 100644 index 0000000..d2a1610 --- /dev/null +++ b/source/serializers.py @@ -0,0 +1,28 @@ +from rest_framework import serializers + +from .constants.source_type import SourceType +from .models import Source + + +class SourceSerializer(serializers.ModelSerializer): + type = serializers.ChoiceField( + choices=SourceType.choices, + help_text=( + "Type of the source. Available options:\n" + "- 'api': Data source accessible through a public API.\n" + "- 'scraper': Data source obtained via web scraping.\n" + "- 'external_tool': Data provided by an external tool." + ), + ) + + class Meta: + model = Source + fields = [ + "id", + "code", + "name", + "type", + "url", + "metrics", + ] + read_only_fields = ["metrics"] diff --git a/source/services.py b/source/services.py new file mode 100644 index 0000000..59685f8 --- /dev/null +++ b/source/services.py @@ -0,0 +1,35 @@ +from source.adapters.base import SourceAdapter + +from .repositories import SourceRepository + +# isort: off +import source.adapters.itunes as _ # noqa +import source.adapters.news as _ # noqa +import source.adapters.reddit as _ # noqa +import source.adapters.google_play_scraper as _ # noqa + +# isort: on + + +class SourceService: + def __init__(self): + self.repo = SourceRepository() + + def list_sources(self): + return self.repo.get_all() + + def get_source(self, source_id): + return self.repo.get_by_id(source_id) + + def create_source(self, validated_data): + return self.repo.create(validated_data) + + def update_source(self, instance, validated_data): + return self.repo.update(instance, validated_data) + + def delete_source(self, instance): + return self.repo.delete(instance) + + @staticmethod + def load_sources(): + return [cls() for cls in SourceAdapter.__subclasses__()] diff --git a/source/services/__init__.py b/source/services/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/source/services/source_registration_service.py b/source/services/source_registration_service.py deleted file mode 100644 index c3bf86b..0000000 --- a/source/services/source_registration_service.py +++ /dev/null @@ -1,32 +0,0 @@ -from source.adapters.base import SourceAdapter -from source.repositories.source_repository import SourceRepository - -# isort: off -import source.adapters.itunes as _ # noqa -import source.adapters.news as _ # noqa -import source.adapters.reddit as _ # noqa -import source.adapters.google_play_scraper as _ # noqa - -# isort: on - - -class SourceRegistrationService: - def __init__(self, repository=None): - self.repository = repository or SourceRepository() - - @staticmethod - def _discover_sources(): - return [cls() for cls in SourceAdapter.__subclasses__()] - - def load_and_register(self): - adapters = self._discover_sources() - for adapter in adapters: - self.repository.get_or_create( - code=adapter.code, - defaults={ - "name": adapter.name, - "type": adapter.type, - "url": adapter.url, - }, - ) - return adapters diff --git a/source/urls.py b/source/urls.py new file mode 100644 index 0000000..a969b6a --- /dev/null +++ b/source/urls.py @@ -0,0 +1,11 @@ +from django.urls import include, path +from rest_framework.routers import DefaultRouter + +from .views import SourceViewSet + +router = DefaultRouter() +router.register(r"sources", SourceViewSet, basename="source") + +urlpatterns = [ + path("", include(router.urls)), +] diff --git a/source/views.py b/source/views.py index e69de29..1817e64 100644 --- a/source/views.py +++ b/source/views.py @@ -0,0 +1,112 @@ +from drf_spectacular.utils import OpenApiParameter, extend_schema +from rest_framework import status, viewsets +from rest_framework.decorators import action +from rest_framework.response import Response + +from metric.serializers import LinkMetricsSerializer + +from .serializers import SourceSerializer +from .services import SourceService + + +class SourceViewSet(viewsets.ViewSet): + service = SourceService() + + @extend_schema( + responses=SourceSerializer(many=True), + tags=["Sources"], + ) + def list(self, request): + sources = self.service.list_sources() + serializer = SourceSerializer(sources, many=True) + return Response(serializer.data) + + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses=SourceSerializer, + tags=["Sources"], + ) + def retrieve(self, request, pk=None): + source = self.service.get_source(pk) + serializer = SourceSerializer(source) + return Response(serializer.data) + + @extend_schema( + request=SourceSerializer, + responses=SourceSerializer, + description=( + "Creates a new data source.\n\n" + "The 'metrics' field is read-only and cannot be set directly on creation." + ), + tags=["Sources"], + ) + def create(self, request): + serializer = SourceSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + source = serializer.save() + return Response(SourceSerializer(source).data, status=status.HTTP_201_CREATED) + + @extend_schema( + request=SourceSerializer, + responses=SourceSerializer, + tags=["Sources"], + ) + def update(self, request, pk=None): + source = self.service.get_source(pk) + serializer = SourceSerializer(source, data=request.data) + serializer.is_valid(raise_exception=True) + updated_source = serializer.save() + return Response(SourceSerializer(updated_source).data) + + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses={204: None}, + tags=["Sources"], + ) + def destroy(self, request, pk=None): + source = self.service.get_source(pk) + source.delete() + return Response(status=status.HTTP_204_NO_CONTENT) + + @extend_schema( + request=LinkMetricsSerializer, + responses=SourceSerializer, + description="Adds a list of metric IDs to an existing " + "source without removing existing ones.", + tags=["Sources"], + ) + @action(detail=True, methods=["post"], url_path="add-metrics") + def add_metrics(self, request, pk=None): + source = self.service.get_source(pk) + metrics_ids = request.data.get("metrics", []) + + if not isinstance(metrics_ids, list): + return Response( + {"error": "metrics must be a list of IDs"}, status=status.HTTP_400_BAD_REQUEST + ) + + source.metrics.add(*metrics_ids) + source.save() + + return Response(SourceSerializer(source).data, status=status.HTTP_200_OK) + + @extend_schema( + request=LinkMetricsSerializer, + responses=SourceSerializer, + description="Removes a list of metric IDs from an existing source.", + tags=["Sources"], + ) + @action(detail=True, methods=["post"], url_path="remove-metrics") + def remove_metrics(self, request, pk=None): + source = self.service.get_source(pk) + metrics_ids = request.data.get("metrics", []) + + if not isinstance(metrics_ids, list): + return Response( + {"error": "metrics must be a list of IDs"}, status=status.HTTP_400_BAD_REQUEST + ) + + source.metrics.remove(*metrics_ids) + source.save() + + return Response(SourceSerializer(source).data, status=status.HTTP_200_OK) diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py index 2414d8a..898c011 100644 --- a/tests/integration/test_end_to_end.py +++ b/tests/integration/test_end_to_end.py @@ -5,7 +5,7 @@ from metric.strategies.generic import GenericMetricStrategy from source.constants.source_type import SourceType from source.models import Source -from source.services.source_registration_service import SourceRegistrationService +from source.services import SourceService @pytest.mark.django_db @@ -26,7 +26,7 @@ def test_full_flow_with_real_adapter(): ) # Adapter real (ja ha de formar part del projecte i registrar-se automàticament) - adapters = SourceRegistrationService().load_and_register() + adapters = SourceService().load_sources() strategy = GenericMetricStrategy("average_rating") values = strategy.compute_all(app.id, adapters) diff --git a/tests/unit/test_source_registration_service.py b/tests/unit/test_source_registration_service.py index fd15b64..ddaa568 100644 --- a/tests/unit/test_source_registration_service.py +++ b/tests/unit/test_source_registration_service.py @@ -1,4 +1,4 @@ -from source.services.source_registration_service import SourceRegistrationService +from source.services import SourceService class FakeRepository: @@ -10,13 +10,15 @@ def get_or_create(self, code, defaults): return None +""" def test_source_registration_service_registers_adapters(): repo = FakeRepository() - service = SourceRegistrationService(repository=repo) + service = SourceService(repository=repo) adapters = service.load_and_register() assert len(repo.registered) == len(adapters) +""" -def test_discover_sources_works(): - adapters = SourceRegistrationService._discover_sources() +def test_load_sources_works(): + adapters = SourceService.load_sources() assert len(adapters) > 0 From 3c194690931e13596ce83312ed8a6fb5b8c9ffe7 Mon Sep 17 00:00:00 2001 From: Anyer Moreno Alcaraz Date: Tue, 29 Apr 2025 10:03:43 +0200 Subject: [PATCH 23/42] Remove tracked __pycache__ files --- config/__pycache__/__init__.cpython-312.pyc | Bin 171 -> 0 bytes config/__pycache__/settings.cpython-312.pyc | Bin 3055 -> 0 bytes config/__pycache__/urls.cpython-312.pyc | Bin 1296 -> 0 bytes config/__pycache__/wsgi.cpython-312.pyc | Bin 657 -> 0 bytes 4 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 config/__pycache__/__init__.cpython-312.pyc delete mode 100644 config/__pycache__/settings.cpython-312.pyc delete mode 100644 config/__pycache__/urls.cpython-312.pyc delete mode 100644 config/__pycache__/wsgi.cpython-312.pyc diff --git a/config/__pycache__/__init__.cpython-312.pyc b/config/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 52dcab13d86e5f2a7207d07eb58fe62a4114d151..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 171 zcmX@j%ge<81n>PHq=V?kAOanHW&w&!XQ*V*Wb|9fP{ah}eFmxd<>hP@6Iz^FR2-9- zSD9KA6Hu9)kyw-)P?VpQnp{#Gu*uC&Da}c>D`Ev2%?QNBAjU^#Mn=XWW*`dyB3XM^$UK|T?;4y zH7Ep0!bjjB3O*zU$X48=8ubr{LxTcYKMH;y@}x;Z3%hHc#DEQlz5UkAh{YRuvaO`I zGU{KoCQz7cof*k^qt;n(H0n_pMIOSNyN?im16&kQ9uCEBF9AC72Iwgi_t34k2HMA4 zl<@o^&`knOe?tm0(9_T-(Ts2kC1G|7rJz0i4Jni8%vXUXDWuS>a2lo2SxlihJ4O&_ z{wtqM3TMy)S_G*DbgtjN)o&S@YzBl`bY4iK-w0>XDPazN^XP)GfPO11?)x&olir3y zrt&RGYwC=NElbv#CbOd(OheapGwV@yjj ztfAyZregvLGvs`qU16^PdJQO5-N1~j0lO+$vaT`kwRUCDCoA|JP>-oy%W9h+X0u3d znCw7@XF)?(vslXlVb;>ya>LAe_GSn4)-GmtI*P(5vS~5;&ggw44S^yo8_XR^>0t9s zbQySOOwoVbLo8=VC#L8U5|M6U?G8)>Rms}5qb$r=Fc`TKgH5pwJI+l0Bd)w&5o>&I zgRhG>_`lh-teLpcG4L|NJ5oonoM4_`-CTEqYm#DOCr~#!*okw+V&xW}7q3@p^_m@B zdG`wY2l%W!C*R3Xc52`w7Ea4Dop)eaapKe6aSux45NZEV`J6PqSBO*o;K zH&ob=nt+&_L|Aa94fiBX$1kl4(T&dOzm2%CV1p+8{?4H?_S%_Y6a{mOK@Gv=F#5>v^`Oo}Bku7}L zreJY6&iI@VUtTYic_&cjO1v=(1;f5RVRt`l+LCGR=>`&^D9Na=j@>k|!C9^&Iu@?U zs;o$EHryOY=!Sjmb(u=CCaWE_h_$A*JHr3zb@78NsSa%%5&Zde3CbN6L%zSv5X9PY z2a3dIJ!gmE>VRy?+KWS4@T96EteEVo+=Oh>*JP}q%sD44LbYv(qT|y|Cp4(7h98{k zgxw0O$lFH1orUnHQ2tFPB8qM?0zlZnrmoz9O4No@qghU5m89@Ing{<=Un|YC(UivuZvZ#R=ZW%$cs0*VgWWQ5TNlQ zTo#+$Ixprbc^(K0C0-ON5L1y&u)MJH$FdW0M|A;=t^;$<)$Szo{2I4etc$>05!VWQ zG5;O$46G-ff0X-h%MC7bOVakSu~pq~>`I1O9bE6`RtZ}Y*KX%@MS<&HH2YBC^U{+LF-3n7`Ny^6*!jcQ`J>o{FXw*&P5VF;7JkmVt1++AyPc~ zh)zCArB9}qr$q2VqUR5%s3*~>V5sL$#C_dpk0Daa2Luf^1rv{w>?1n;;OZfL@k!86 zgG{h5Ge<@_pH#;ue;J=_3?`@J3%&y)-TOT;lkUb(re?a)lh|}OauSVq WL%k^?F$;!L#F_UGiKOu!c>90eLAUh) diff --git a/config/__pycache__/urls.cpython-312.pyc b/config/__pycache__/urls.cpython-312.pyc deleted file mode 100644 index 9c0c174aa54786227897f3eaf1408614f6936b8c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1296 zcmb7E&ui2`6rNSHVYu1w4MdslCo!-_RUo`QpUcl{=IS=8~n15TvRBHUp-+rcqsedW5oxU6H`;Vi`AuubAP+Z@wO{V!(zDg%|kM27GwB-@*-)#kIjdMu># zI3gjvQd4%#GM54(#KI;d=1rjfjRliE$`uhiWt`Jl0j&RFqx@|m+6rXO`h(GG4O2B3WbG# zHkRtGyMl7jBGHiY*6pF&BRsr49t+X>Gr+?2ae&r%qFPEoJ02#6Y8S5l3n$?cN8$Dy zeXgGRFmvN$b@_v_d;mM>Nh#IANzj^AP0H;yj7Gbi9L=f$n3DUOTuG2rCrZSP$sIK+ zyaYRGyu=W`q=})JB-M%b-a)*ooTlLRvvN6P-XNe?x#FjC7mINFR?swUUq7Z9KQPi} zzo64!(7Er1rZu$v3Mx)*&Ah%es&Cgvi`$DIjkBMO%GQakjhAcRiiKnPi)tF+7u!H~ AuK)l5 diff --git a/config/__pycache__/wsgi.cpython-312.pyc b/config/__pycache__/wsgi.cpython-312.pyc deleted file mode 100644 index 366040654857935f5f862a78376c6d6c914590aa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 657 zcmYjP&u`N(6tL8T z(?bJ3pa}I)gd>aL9yYtptkmw=7y%y5&suKxQSBDbP*jQTom*vwk%hoMzbWt3D zF_p=6iPID!((ltOV@XJLB6z=V=It1gR7fBRH+<=Jfe3DfCyERb2G58BaE4lEYTff9 z5h^ztQ67t#rck6FaPPUh<_*P(7G4Ay6O$CU_D)osc+T(QmF+stAIG4B;w-o$BVXQn z)o3?6L3ew9ztwJbgWb;7!RzhQ2c-|L0<9Ast9muB27yr!1i2+t{;<^HhDN?yOqI6b z$*aJ}i4;6{Ok)HCddzfQyF>U9O$$L+JDcnNfdZ*~$|oTGy-7HvGTke^R{kz%+Q_m^ z!I<%0$dwHm9s!U1QkP#)y4mDv{zYl|OSx#qnqP#}s#*Tp7~_9d6<4N@5O#i{wM%5Z hbIzRaXz4Rrx>$BERvTZ@#*f7(UlyO9)#QEC^dIKx!!!T@ From 7d8dbd9a51e18cdbf671866d9b5ce8e8e36046de Mon Sep 17 00:00:00 2001 From: Anyer Moreno Alcaraz Date: Tue, 29 Apr 2025 10:24:07 +0200 Subject: [PATCH 24/42] Added tag Apps --- app/views.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/app/views.py b/app/views.py index 748f184..6ce18c7 100644 --- a/app/views.py +++ b/app/views.py @@ -9,7 +9,10 @@ class AppViewSet(viewsets.ViewSet): service = AppService() - @extend_schema(responses=AppSerializer(many=True)) + @extend_schema( + responses=AppSerializer(many=True), + tags=["Apps"], + ) def list(self, request): apps = self.service.list_apps() serializer = AppSerializer(apps, many=True) @@ -18,6 +21,7 @@ def list(self, request): @extend_schema( parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], responses=AppSerializer, + tags=["Apps"], ) def retrieve(self, request, pk=None): app = self.service.get_app(pk) @@ -35,6 +39,7 @@ def retrieve(self, request, pk=None): "on the provided identifiers.\n\n" "You do not need to manually provide these fields when creating an app." ), + tags=["Apps"], ) def create(self, request): print("REQUEST DATA:", request.data) @@ -43,7 +48,11 @@ def create(self, request): app = self.service.create_app(serializer.validated_data) return Response(AppSerializer(app).data, status=status.HTTP_201_CREATED) - @extend_schema(request=AppSerializer, responses=AppSerializer) + @extend_schema( + request=AppSerializer, + responses=AppSerializer, + tags=["Apps"], + ) def update(self, request, pk=None): app = self.service.get_app(pk) serializer = AppSerializer(app, data=request.data) @@ -54,6 +63,7 @@ def update(self, request, pk=None): @extend_schema( parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], responses={204: None}, + tags=["Apps"], ) def destroy(self, request, pk=None): app = self.service.get_app(pk) From 789ad02fe12c281b96e6676a19195521587d4af1 Mon Sep 17 00:00:00 2001 From: Anyer Moreno Alcaraz Date: Tue, 29 Apr 2025 13:39:58 +0200 Subject: [PATCH 25/42] =?UTF-8?q?feat(api-rest):=20Crear=20les=20views=20i?= =?UTF-8?q?=20urls=20de=20l=E2=80=99app=20review=20i=20documentar-ho=20Ref?= =?UTF-8?q?s:=20#67?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/urls.py | 1 + ..._app_version_review_replied_at_and_more.py | 33 +++++++ review/models.py | 4 + review/repositories.py | 74 +++++++++++++++ review/serializers.py | 42 +++++++++ review/services.py | 28 ++++++ review/urls.py | 11 +++ review/views.py | 89 +++++++++++++++++++ source/adapters/google_play_scraper.py | 54 ++++++++--- 9 files changed, 325 insertions(+), 11 deletions(-) create mode 100644 review/migrations/0002_review_app_version_review_replied_at_and_more.py create mode 100644 review/repositories.py create mode 100644 review/serializers.py create mode 100644 review/services.py create mode 100644 review/urls.py diff --git a/config/urls.py b/config/urls.py index 704e0a6..9aed731 100644 --- a/config/urls.py +++ b/config/urls.py @@ -32,4 +32,5 @@ def home(request): path("api/docs/", SpectacularSwaggerView.as_view(url_name="schema"), name="swagger-ui"), path("api/", include("app.urls")), path("api/", include("source.urls")), + path("api/", include("review.urls")), ] diff --git a/review/migrations/0002_review_app_version_review_replied_at_and_more.py b/review/migrations/0002_review_app_version_review_replied_at_and_more.py new file mode 100644 index 0000000..ab7e30a --- /dev/null +++ b/review/migrations/0002_review_app_version_review_replied_at_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 5.1.7 on 2025-04-29 09:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("review", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="review", + name="app_version", + field=models.CharField(blank=True, max_length=100, null=True), + ), + migrations.AddField( + model_name="review", + name="replied_at", + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name="review", + name="reply_content", + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name="review", + name="review_id", + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/review/models.py b/review/models.py index eef4159..3616b07 100644 --- a/review/models.py +++ b/review/models.py @@ -29,6 +29,7 @@ class ReviewTopic(models.TextChoices): class Review(models.Model): + review_id = models.CharField(max_length=255, null=True, blank=True) app = models.ForeignKey(App, on_delete=models.CASCADE) author = models.CharField(max_length=100, null=True, blank=True) content = models.TextField() @@ -39,6 +40,9 @@ class Review(models.Model): type = models.CharField(max_length=20, choices=ReviewType.choices) topic = models.CharField(max_length=30, choices=ReviewTopic.choices, null=True, blank=True) date = models.DateTimeField() + reply_content = models.TextField(null=True, blank=True) + replied_at = models.DateTimeField(null=True, blank=True) + app_version = models.CharField(max_length=100, null=True, blank=True) def __str__(self): return f"{self.app.name} - {self.date.date()} - {self.author or 'Anon'}" diff --git a/review/repositories.py b/review/repositories.py new file mode 100644 index 0000000..95a98e7 --- /dev/null +++ b/review/repositories.py @@ -0,0 +1,74 @@ +from datetime import datetime + +from django.shortcuts import get_object_or_404 + +from .models import Review + + +class ReviewRepository: + def get_all(self, filters=None): + queryset = Review.objects.all() + + if filters: + app_id = filters.get("app") + date_from = filters.get("date_from") + date_to = filters.get("date_to") + + if app_id: + queryset = queryset.filter(app_id=app_id) + + if date_from: + try: + queryset = queryset.filter(date__gte=datetime.fromisoformat(date_from)) + except ValueError: + pass + + if date_to: + try: + queryset = queryset.filter(date__lte=datetime.fromisoformat(date_to)) + except ValueError: + pass + + return queryset + + def get_by_id(self, review_id): + return get_object_or_404(Review, id=review_id) + + def save_reviews(self, app_id, reviews_list): + if not reviews_list: + return 0 + + # Agafar tots els review_ids existents d'aquesta app + existing_review_ids = set( + Review.objects.filter( + review_id__in=[r.get("reviewId") for r in reviews_list] + ).values_list("review_id", flat=True) + ) + + created_reviews = [] + + for r in reviews_list: + review_id = r.get("reviewId") + if not review_id or review_id in existing_review_ids: + continue + + review = Review( + app_id=app_id, + review_id=review_id, + author=r.get("userName"), + content=r.get("content"), + rating=r.get("score"), + # polarity=self._infer_polarity(r.get('score')), + # type="general", + # topic=None, + date=r.get("at"), + reply_content=r.get("replyContent"), + replied_at=r.get("repliedAt"), + app_version=r.get("appVersion"), + ) + created_reviews.append(review) + + Review.objects.bulk_create(created_reviews) + print(f"Saved {len(created_reviews)} new reviews.") + + return len(created_reviews) diff --git a/review/serializers.py b/review/serializers.py new file mode 100644 index 0000000..0a84f20 --- /dev/null +++ b/review/serializers.py @@ -0,0 +1,42 @@ +from rest_framework import serializers + +from .models import Review, ReviewPolarity, ReviewTopic, ReviewType + + +class ReviewSerializer(serializers.ModelSerializer): + polarity = serializers.ChoiceField(choices=ReviewPolarity.choices) + type = serializers.ChoiceField(choices=ReviewType.choices) + topic = serializers.ChoiceField(choices=ReviewTopic.choices) + + class Meta: + model = Review + fields = [ + "id", + "review_id", + "app", + "author", + "content", + "rating", + "polarity", + "type", + "topic", + "date", + "reply_content", + "replied_at", + "app_version", + ] + read_only_fields = [ + "id", + "review_id", + "app", + "author", + "content", + "rating", + "polarity", + "type", + "topic", + "date", + "reply_content", + "replied_at", + "app_version", + ] diff --git a/review/services.py b/review/services.py new file mode 100644 index 0000000..7697bd1 --- /dev/null +++ b/review/services.py @@ -0,0 +1,28 @@ +from source.adapters.google_play_scraper import GooglePlayScraperAdapter + +from .repositories import ReviewRepository + + +class ReviewService: + def __init__(self): + self.repo = ReviewRepository() + self.adapter = GooglePlayScraperAdapter() + + def list_reviews(self, filters=None): + return self.repo.get_all(filters=filters) + + def get_review(self, review_id): + return self.repo.get_by_id(review_id) + + def poll_reviews(self, app_id, date_from=None, date_to=None, max_reviews=2000): + reviews = self.adapter.fetch_reviews( + app_id=app_id, date_from=date_from, date_to=date_to, max_reviews=max_reviews + ) + + saved_count = self.repo.save_reviews(app_id, reviews) + + return { + "fetched": len(reviews), + "saved": saved_count, + "reviews": reviews, + } diff --git a/review/urls.py b/review/urls.py new file mode 100644 index 0000000..f43b970 --- /dev/null +++ b/review/urls.py @@ -0,0 +1,11 @@ +from django.urls import include, path +from rest_framework.routers import DefaultRouter + +from .views import ReviewViewSet + +router = DefaultRouter() +router.register(r"reviews", ReviewViewSet, basename="review") + +urlpatterns = [ + path("", include(router.urls)), +] diff --git a/review/views.py b/review/views.py index e69de29..a0a668a 100644 --- a/review/views.py +++ b/review/views.py @@ -0,0 +1,89 @@ +from drf_spectacular.utils import OpenApiParameter, extend_schema +from rest_framework import status, viewsets +from rest_framework.decorators import action +from rest_framework.response import Response + +from .serializers import ReviewSerializer +from .services import ReviewService + + +class ReviewViewSet(viewsets.ViewSet): + service = ReviewService() + + @extend_schema( + parameters=[ + OpenApiParameter(name="app", type=int, description="Filter by app ID"), + OpenApiParameter(name="date_from", type=str, description="Start date (YYYY-MM-DD)"), + OpenApiParameter(name="date_to", type=str, description="End date (YYYY-MM-DD)"), + ], + responses=ReviewSerializer(many=True), + tags=["Reviews"], + ) + def list(self, request): + filters = { + "app": request.query_params.get("app"), + "date_from": request.query_params.get("date_from"), + "date_to": request.query_params.get("date_to"), + } + reviews = self.service.list_reviews(filters) + serializer = ReviewSerializer(reviews, many=True) + return Response(serializer.data) + + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses=ReviewSerializer, + tags=["Reviews"], + ) + def retrieve(self, request, pk=None): + source = self.service.get_source(pk) + serializer = ReviewSerializer(source) + return Response(serializer.data) + + @extend_schema( + request=None, + parameters=[ + OpenApiParameter( + name="app", type=int, required=True, description="App ID to fetch reviews for" + ), + OpenApiParameter( + name="date_from", type=str, description="Start date (YYYY-MM-DD)", required=False + ), + OpenApiParameter( + name="date_to", type=str, description="End date (YYYY-MM-DD)", required=False + ), + OpenApiParameter( + name="max_reviews", + type=int, + description="Maximum number of reviews to fetch", + required=False, + ), + ], + responses={"200": {"type": "object"}}, + tags=["Reviews"], + description="Polls Google Play for reviews of a given app between two dates, " + "saves new ones in the database, and returns the fetched reviews and save stats.", + ) + @action(detail=False, methods=["post"], url_path="poll-reviews") + def poll_reviews(self, request): + app_id = request.query_params.get("app") + date_from = request.query_params.get("date_from") + date_to = request.query_params.get("date_to") + max_reviews = request.query_params.get("max_reviews") + + if not app_id: + return Response( + {"error": "The 'app' parameter is required."}, status=status.HTTP_400_BAD_REQUEST + ) + + try: + max_reviews = int(max_reviews) if max_reviews else 2000 + except ValueError: + return Response( + {"error": "max_reviews must be an integer."}, status=status.HTTP_400_BAD_REQUEST + ) + + result = self.service.poll_reviews( + app_id=int(app_id), date_from=date_from, date_to=date_to, max_reviews=max_reviews + ) + + return Response(result) diff --git a/source/adapters/google_play_scraper.py b/source/adapters/google_play_scraper.py index d92c994..2c9026a 100644 --- a/source/adapters/google_play_scraper.py +++ b/source/adapters/google_play_scraper.py @@ -1,6 +1,6 @@ -from datetime import timedelta +import time +from datetime import datetime -from django.utils import timezone from google_play_scraper import app as gp_app from google_play_scraper import reviews @@ -54,23 +54,55 @@ def lookup_app(package_name: str) -> dict | None: print(f"[GooglePlayScraperAdapter] Error fetching app data: {e}") return None - def fetch_reviews(self, app_id: int): + def fetch_reviews(self, app_id: int, date_from=None, date_to=None, max_reviews=2000): try: app_instance = App.objects.get(id=app_id) package_name = app_instance.playstore_id except App.DoesNotExist: print(f"[GooglePlayScraperAdapter] App with id {app_id} does not exist.") - return None + return [] except AttributeError: print("[GooglePlayScraperAdapter] App is missing 'playstore_id' field.") - return None + return [] + + all_reviews = [] + next_token = None + + while True: + result, next_token = reviews( + package_name, lang="en", country="us", count=200, continuation_token=next_token + ) + + if not result: + break + + all_reviews.extend(result) + + if len(all_reviews) >= max_reviews: + print(f"Reached maximum limit of {max_reviews} reviews.") + break + + if not next_token: + break # No more reviews to fetch + + time.sleep(1) # Opcional: per evitar ser bloquejats - yesterday = timezone.now().date() - timedelta(days=1) + # Filtrar per dates si cal + if date_from: + date_from = datetime.fromisoformat(date_from).date() + if date_to: + date_to = datetime.fromisoformat(date_to).date() - result, _ = reviews(package_name, count=200) + reviews_filtered = [] + for r in all_reviews: + review_date = r["at"].date() + if date_from and review_date < date_from: + continue + if date_to and review_date > date_to: + continue + reviews_filtered.append(r) - reviews_from_yesterday = [r for r in result if r["at"].date() == yesterday] + print(f"Total reviews collected: {len(all_reviews)}") + print(f"Reviews after filtering: {len(reviews_filtered)}") - print(f"Ressenyes d'ahir: {reviews_from_yesterday}") - print(f"Ressenyes d'ahir: {len(reviews_from_yesterday)}") - return reviews_from_yesterday + return reviews_filtered From 731e7065e5e4984350254f2bba29f9d3c2bf5e82 Mon Sep 17 00:00:00 2001 From: Anyer Date: Tue, 29 Apr 2025 16:58:03 +0200 Subject: [PATCH 26/42] Remove tracked __pycache__ files --- config/__pycache__/__init__.cpython-312.pyc | Bin 171 -> 0 bytes config/__pycache__/settings.cpython-312.pyc | Bin 3055 -> 0 bytes config/__pycache__/urls.cpython-312.pyc | Bin 1296 -> 0 bytes config/__pycache__/wsgi.cpython-312.pyc | Bin 657 -> 0 bytes 4 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 config/__pycache__/__init__.cpython-312.pyc delete mode 100644 config/__pycache__/settings.cpython-312.pyc delete mode 100644 config/__pycache__/urls.cpython-312.pyc delete mode 100644 config/__pycache__/wsgi.cpython-312.pyc diff --git a/config/__pycache__/__init__.cpython-312.pyc b/config/__pycache__/__init__.cpython-312.pyc deleted file mode 100644 index 52dcab13d86e5f2a7207d07eb58fe62a4114d151..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 171 zcmX@j%ge<81n>PHq=V?kAOanHW&w&!XQ*V*Wb|9fP{ah}eFmxd<>hP@6Iz^FR2-9- zSD9KA6Hu9)kyw-)P?VpQnp{#Gu*uC&Da}c>D`Ev2%?QNBAjU^#Mn=XWW*`dyB3XM^$UK|T?;4y zH7Ep0!bjjB3O*zU$X48=8ubr{LxTcYKMH;y@}x;Z3%hHc#DEQlz5UkAh{YRuvaO`I zGU{KoCQz7cof*k^qt;n(H0n_pMIOSNyN?im16&kQ9uCEBF9AC72Iwgi_t34k2HMA4 zl<@o^&`knOe?tm0(9_T-(Ts2kC1G|7rJz0i4Jni8%vXUXDWuS>a2lo2SxlihJ4O&_ z{wtqM3TMy)S_G*DbgtjN)o&S@YzBl`bY4iK-w0>XDPazN^XP)GfPO11?)x&olir3y zrt&RGYwC=NElbv#CbOd(OheapGwV@yjj ztfAyZregvLGvs`qU16^PdJQO5-N1~j0lO+$vaT`kwRUCDCoA|JP>-oy%W9h+X0u3d znCw7@XF)?(vslXlVb;>ya>LAe_GSn4)-GmtI*P(5vS~5;&ggw44S^yo8_XR^>0t9s zbQySOOwoVbLo8=VC#L8U5|M6U?G8)>Rms}5qb$r=Fc`TKgH5pwJI+l0Bd)w&5o>&I zgRhG>_`lh-teLpcG4L|NJ5oonoM4_`-CTEqYm#DOCr~#!*okw+V&xW}7q3@p^_m@B zdG`wY2l%W!C*R3Xc52`w7Ea4Dop)eaapKe6aSux45NZEV`J6PqSBO*o;K zH&ob=nt+&_L|Aa94fiBX$1kl4(T&dOzm2%CV1p+8{?4H?_S%_Y6a{mOK@Gv=F#5>v^`Oo}Bku7}L zreJY6&iI@VUtTYic_&cjO1v=(1;f5RVRt`l+LCGR=>`&^D9Na=j@>k|!C9^&Iu@?U zs;o$EHryOY=!Sjmb(u=CCaWE_h_$A*JHr3zb@78NsSa%%5&Zde3CbN6L%zSv5X9PY z2a3dIJ!gmE>VRy?+KWS4@T96EteEVo+=Oh>*JP}q%sD44LbYv(qT|y|Cp4(7h98{k zgxw0O$lFH1orUnHQ2tFPB8qM?0zlZnrmoz9O4No@qghU5m89@Ing{<=Un|YC(UivuZvZ#R=ZW%$cs0*VgWWQ5TNlQ zTo#+$Ixprbc^(K0C0-ON5L1y&u)MJH$FdW0M|A;=t^;$<)$Szo{2I4etc$>05!VWQ zG5;O$46G-ff0X-h%MC7bOVakSu~pq~>`I1O9bE6`RtZ}Y*KX%@MS<&HH2YBC^U{+LF-3n7`Ny^6*!jcQ`J>o{FXw*&P5VF;7JkmVt1++AyPc~ zh)zCArB9}qr$q2VqUR5%s3*~>V5sL$#C_dpk0Daa2Luf^1rv{w>?1n;;OZfL@k!86 zgG{h5Ge<@_pH#;ue;J=_3?`@J3%&y)-TOT;lkUb(re?a)lh|}OauSVq WL%k^?F$;!L#F_UGiKOu!c>90eLAUh) diff --git a/config/__pycache__/urls.cpython-312.pyc b/config/__pycache__/urls.cpython-312.pyc deleted file mode 100644 index 9c0c174aa54786227897f3eaf1408614f6936b8c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1296 zcmb7E&ui2`6rNSHVYu1w4MdslCo!-_RUo`QpUcl{=IS=8~n15TvRBHUp-+rcqsedW5oxU6H`;Vi`AuubAP+Z@wO{V!(zDg%|kM27GwB-@*-)#kIjdMu># zI3gjvQd4%#GM54(#KI;d=1rjfjRliE$`uhiWt`Jl0j&RFqx@|m+6rXO`h(GG4O2B3WbG# zHkRtGyMl7jBGHiY*6pF&BRsr49t+X>Gr+?2ae&r%qFPEoJ02#6Y8S5l3n$?cN8$Dy zeXgGRFmvN$b@_v_d;mM>Nh#IANzj^AP0H;yj7Gbi9L=f$n3DUOTuG2rCrZSP$sIK+ zyaYRGyu=W`q=})JB-M%b-a)*ooTlLRvvN6P-XNe?x#FjC7mINFR?swUUq7Z9KQPi} zzo64!(7Er1rZu$v3Mx)*&Ah%es&Cgvi`$DIjkBMO%GQakjhAcRiiKnPi)tF+7u!H~ AuK)l5 diff --git a/config/__pycache__/wsgi.cpython-312.pyc b/config/__pycache__/wsgi.cpython-312.pyc deleted file mode 100644 index 366040654857935f5f862a78376c6d6c914590aa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 657 zcmYjP&u`N(6tL8T z(?bJ3pa}I)gd>aL9yYtptkmw=7y%y5&suKxQSBDbP*jQTom*vwk%hoMzbWt3D zF_p=6iPID!((ltOV@XJLB6z=V=It1gR7fBRH+<=Jfe3DfCyERb2G58BaE4lEYTff9 z5h^ztQ67t#rck6FaPPUh<_*P(7G4Ay6O$CU_D)osc+T(QmF+stAIG4B;w-o$BVXQn z)o3?6L3ew9ztwJbgWb;7!RzhQ2c-|L0<9Ast9muB27yr!1i2+t{;<^HhDN?yOqI6b z$*aJ}i4;6{Ok)HCddzfQyF>U9O$$L+JDcnNfdZ*~$|oTGy-7HvGTke^R{kz%+Q_m^ z!I<%0$dwHm9s!U1QkP#)y4mDv{zYl|OSx#qnqP#}s#*Tp7~_9d6<4N@5O#i{wM%5Z hbIzRaXz4Rrx>$BERvTZ@#*f7(UlyO9)#QEC^dIKx!!!T@ From 2188ea5c2e9532451fa2e87a2aa5150f39491cac Mon Sep 17 00:00:00 2001 From: Anyer Date: Tue, 29 Apr 2025 19:15:31 +0200 Subject: [PATCH 27/42] =?UTF-8?q?feature(api-rest):=20Crear=20les=20views?= =?UTF-8?q?=20i=20urls=20de=20l=E2=80=99app=20review=20i=20documentar-ho?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #67 --- review/repositories.py | 13 ++++- review/services.py | 6 +-- review/views.py | 4 +- source/adapters/google_play_scraper.py | 74 +++++++++++++++----------- 4 files changed, 58 insertions(+), 39 deletions(-) diff --git a/review/repositories.py b/review/repositories.py index 95a98e7..21b8ff0 100644 --- a/review/repositories.py +++ b/review/repositories.py @@ -1,6 +1,7 @@ from datetime import datetime from django.shortcuts import get_object_or_404 +from django.utils import timezone from .models import Review @@ -52,6 +53,14 @@ def save_reviews(self, app_id, reviews_list): if not review_id or review_id in existing_review_ids: continue + created_at = r.get("at") + if created_at and timezone.is_naive(created_at): + created_at = timezone.make_aware(created_at) + + replied_at = r.get("repliedAt") + if replied_at and timezone.is_naive(replied_at): + replied_at = timezone.make_aware(replied_at) + review = Review( app_id=app_id, review_id=review_id, @@ -61,9 +70,9 @@ def save_reviews(self, app_id, reviews_list): # polarity=self._infer_polarity(r.get('score')), # type="general", # topic=None, - date=r.get("at"), + date=created_at, reply_content=r.get("replyContent"), - replied_at=r.get("repliedAt"), + replied_at=replied_at, app_version=r.get("appVersion"), ) created_reviews.append(review) diff --git a/review/services.py b/review/services.py index 7697bd1..22a3a8b 100644 --- a/review/services.py +++ b/review/services.py @@ -14,10 +14,8 @@ def list_reviews(self, filters=None): def get_review(self, review_id): return self.repo.get_by_id(review_id) - def poll_reviews(self, app_id, date_from=None, date_to=None, max_reviews=2000): - reviews = self.adapter.fetch_reviews( - app_id=app_id, date_from=date_from, date_to=date_to, max_reviews=max_reviews - ) + def poll_reviews(self, app_id, date_from=None, date_to=None): + reviews = self.adapter.fetch_reviews(app_id=app_id, date_from=date_from, date_to=date_to) saved_count = self.repo.save_reviews(app_id, reviews) diff --git a/review/views.py b/review/views.py index a0a668a..913c38b 100644 --- a/review/views.py +++ b/review/views.py @@ -82,8 +82,6 @@ def poll_reviews(self, request): {"error": "max_reviews must be an integer."}, status=status.HTTP_400_BAD_REQUEST ) - result = self.service.poll_reviews( - app_id=int(app_id), date_from=date_from, date_to=date_to, max_reviews=max_reviews - ) + result = self.service.poll_reviews(app_id=int(app_id), date_from=date_from, date_to=date_to) return Response(result) diff --git a/source/adapters/google_play_scraper.py b/source/adapters/google_play_scraper.py index 2c9026a..8a22114 100644 --- a/source/adapters/google_play_scraper.py +++ b/source/adapters/google_play_scraper.py @@ -1,5 +1,5 @@ import time -from datetime import datetime +from datetime import date, datetime from google_play_scraper import app as gp_app from google_play_scraper import reviews @@ -54,7 +54,7 @@ def lookup_app(package_name: str) -> dict | None: print(f"[GooglePlayScraperAdapter] Error fetching app data: {e}") return None - def fetch_reviews(self, app_id: int, date_from=None, date_to=None, max_reviews=2000): + def fetch_reviews(self, app_id: int, date_from=None, date_to=None): try: app_instance = App.objects.get(id=app_id) package_name = app_instance.playstore_id @@ -65,44 +65,58 @@ def fetch_reviews(self, app_id: int, date_from=None, date_to=None, max_reviews=2 print("[GooglePlayScraperAdapter] App is missing 'playstore_id' field.") return [] + if not package_name: + print("[GooglePlayScraperAdapter] playstore_id is None.") + return [] + + date_from = datetime.fromisoformat(date_from).date() if date_from else None + + if date_to: + date_to = datetime.fromisoformat(date_to).date() + elif date_from: + date_to = date.today() + + days_range = (date_to - date_from).days if date_from and date_to else None + + if days_range is not None and days_range <= 1: + count_per_request = 200 + elif days_range is not None and days_range <= 7: + count_per_request = 1000 + else: + count_per_request = 4500 all_reviews = [] next_token = None while True: - result, next_token = reviews( - package_name, lang="en", country="us", count=200, continuation_token=next_token + batch, next_token = reviews( + package_name, + lang="en", + country="us", + count=count_per_request, # màxim per petició + continuation_token=next_token, ) - if not result: + if not batch: break - all_reviews.extend(result) + for r in batch: + review_date = r["at"].date() - if len(all_reviews) >= max_reviews: - print(f"Reached maximum limit of {max_reviews} reviews.") - break + # Si la review està dins del rang, l'afegim + if (not date_from or review_date >= date_from) and ( + not date_to or review_date <= date_to + ): + all_reviews.append(r) - if not next_token: - break # No more reviews to fetch - - time.sleep(1) # Opcional: per evitar ser bloquejats + # Si totes les següents seran més antigues que `date_from`, podem parar + elif date_from and review_date < date_from: + print("Sortint perquè ja hem passat per sota de date_from.") + print(len(all_reviews), "revisions recollides.") + return all_reviews # Ja no cal continuar - # Filtrar per dates si cal - if date_from: - date_from = datetime.fromisoformat(date_from).date() - if date_to: - date_to = datetime.fromisoformat(date_to).date() - - reviews_filtered = [] - for r in all_reviews: - review_date = r["at"].date() - if date_from and review_date < date_from: - continue - if date_to and review_date > date_to: - continue - reviews_filtered.append(r) + if not next_token: + break - print(f"Total reviews collected: {len(all_reviews)}") - print(f"Reviews after filtering: {len(reviews_filtered)}") + time.sleep(1) # per evitar bloquejos - return reviews_filtered + return all_reviews From ec0656b22feb41eb18e4ab49dd17edf6102296f9 Mon Sep 17 00:00:00 2001 From: Anyerrr Date: Wed, 30 Apr 2025 10:22:41 +0200 Subject: [PATCH 28/42] feat(api-rest): post de poll_reviews manual --- review/urls.py | 5 +++ review/views.py | 47 +++++++++------------ source/adapters/google_play_scraper.py | 57 ++++++++++++-------------- 3 files changed, 50 insertions(+), 59 deletions(-) diff --git a/review/urls.py b/review/urls.py index f43b970..613ca2e 100644 --- a/review/urls.py +++ b/review/urls.py @@ -8,4 +8,9 @@ urlpatterns = [ path("", include(router.urls)), + path( + "apps//reviews/polling/", + ReviewViewSet.as_view({"post": "poll_reviews"}), + name="poll-reviews" + ), ] diff --git a/review/views.py b/review/views.py index 913c38b..da438ae 100644 --- a/review/views.py +++ b/review/views.py @@ -1,4 +1,4 @@ -from drf_spectacular.utils import OpenApiParameter, extend_schema +from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_schema from rest_framework import status, viewsets from rest_framework.decorators import action from rest_framework.response import Response @@ -43,7 +43,7 @@ def retrieve(self, request, pk=None): request=None, parameters=[ OpenApiParameter( - name="app", type=int, required=True, description="App ID to fetch reviews for" + name="id", type=int, required=True, location="path", description="App ID to fetch reviews for" ), OpenApiParameter( name="date_from", type=str, description="Start date (YYYY-MM-DD)", required=False @@ -51,37 +51,28 @@ def retrieve(self, request, pk=None): OpenApiParameter( name="date_to", type=str, description="End date (YYYY-MM-DD)", required=False ), - OpenApiParameter( - name="max_reviews", - type=int, - description="Maximum number of reviews to fetch", - required=False, - ), ], - responses={"200": {"type": "object"}}, + responses={ + 200: OpenApiResponse(description="Reviews successfully fetched and saved."), + 400: OpenApiResponse( + description="Invalid request. Possible causes: missing parameters, " + "wrong date format, or invalid date range." + ), + 404: OpenApiResponse(description="App with the given ID was not found."), + }, tags=["Reviews"], - description="Polls Google Play for reviews of a given app between two dates, " - "saves new ones in the database, and returns the fetched reviews and save stats.", + description=( + "Polls Google Play for reviews of a given app between two dates.\n\n" + "Saves new ones in the database and returns" + "the fetched reviews with stats.\n\n" + "If no `date_from` and `date_to` are provided," + " reviews from **yesterday** will be fetched." + ), ) - @action(detail=False, methods=["post"], url_path="poll-reviews") - def poll_reviews(self, request): - app_id = request.query_params.get("app") + def poll_reviews(self, request, pk=None): date_from = request.query_params.get("date_from") date_to = request.query_params.get("date_to") - max_reviews = request.query_params.get("max_reviews") - - if not app_id: - return Response( - {"error": "The 'app' parameter is required."}, status=status.HTTP_400_BAD_REQUEST - ) - - try: - max_reviews = int(max_reviews) if max_reviews else 2000 - except ValueError: - return Response( - {"error": "max_reviews must be an integer."}, status=status.HTTP_400_BAD_REQUEST - ) - result = self.service.poll_reviews(app_id=int(app_id), date_from=date_from, date_to=date_to) + result = self.service.poll_reviews(app_id=int(pk), date_from=date_from, date_to=date_to) return Response(result) diff --git a/source/adapters/google_play_scraper.py b/source/adapters/google_play_scraper.py index 8a22114..a377b3f 100644 --- a/source/adapters/google_play_scraper.py +++ b/source/adapters/google_play_scraper.py @@ -1,8 +1,9 @@ import time -from datetime import date, datetime +from datetime import date, datetime, timedelta from google_play_scraper import app as gp_app from google_play_scraper import reviews +from rest_framework.exceptions import NotFound, ParseError, ValidationError from app.models import App from metric.constants import MetricCodes @@ -59,42 +60,40 @@ def fetch_reviews(self, app_id: int, date_from=None, date_to=None): app_instance = App.objects.get(id=app_id) package_name = app_instance.playstore_id except App.DoesNotExist: - print(f"[GooglePlayScraperAdapter] App with id {app_id} does not exist.") - return [] + raise NotFound(f"App with id {app_id} does not exist.") except AttributeError: - print("[GooglePlayScraperAdapter] App is missing 'playstore_id' field.") - return [] + raise ParseError("App is missing 'playstore_id' field.") if not package_name: - print("[GooglePlayScraperAdapter] playstore_id is None.") - return [] + raise ParseError("App has no 'playstore_id' set.") - date_from = datetime.fromisoformat(date_from).date() if date_from else None + # Dates per defecte: ahir + yesterday = date.today() - timedelta(days=1) + try: + date_from = datetime.fromisoformat(date_from).date() if date_from else yesterday + date_to = datetime.fromisoformat(date_to).date() if date_to else yesterday + except ValueError: + raise ParseError("Dates must be in ISO format: YYYY-MM-DD.") - if date_to: - date_to = datetime.fromisoformat(date_to).date() - elif date_from: - date_to = date.today() + days_range = (date_to - date_from).days + if days_range < 0: + raise ValidationError("date_from must be before or equal to date_to.") - days_range = (date_to - date_from).days if date_from and date_to else None + count_per_request = min(180 * days_range, 4500) - if days_range is not None and days_range <= 1: - count_per_request = 200 - elif days_range is not None and days_range <= 7: - count_per_request = 1000 - else: - count_per_request = 4500 all_reviews = [] next_token = None + count = 0 while True: batch, next_token = reviews( package_name, lang="en", country="us", - count=count_per_request, # màxim per petició + count=count_per_request, continuation_token=next_token, ) + count += 1 if not batch: break @@ -102,21 +101,17 @@ def fetch_reviews(self, app_id: int, date_from=None, date_to=None): for r in batch: review_date = r["at"].date() - # Si la review està dins del rang, l'afegim - if (not date_from or review_date >= date_from) and ( - not date_to or review_date <= date_to - ): - all_reviews.append(r) + if review_date < date_from: + print("⛔ Hem arribat a reviews més antigues del rang. Sortint.") + return all_reviews - # Si totes les següents seran més antigues que `date_from`, podem parar - elif date_from and review_date < date_from: - print("Sortint perquè ja hem passat per sota de date_from.") - print(len(all_reviews), "revisions recollides.") - return all_reviews # Ja no cal continuar + if review_date <= date_to: + all_reviews.append(r) if not next_token: break - time.sleep(1) # per evitar bloquejos + time.sleep(1) + print(f"✅ Total reviews: {len(all_reviews)} recollides en {count} peticions.") return all_reviews From 2b3371cd71f36ae1cfe31f6d8340e20fe3d1c7ce Mon Sep 17 00:00:00 2001 From: Anyerrr Date: Wed, 30 Apr 2025 10:26:17 +0200 Subject: [PATCH 29/42] feat(api-rest): Import no usats --- review/urls.py | 2 +- review/views.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/review/urls.py b/review/urls.py index 613ca2e..cec6f9c 100644 --- a/review/urls.py +++ b/review/urls.py @@ -11,6 +11,6 @@ path( "apps//reviews/polling/", ReviewViewSet.as_view({"post": "poll_reviews"}), - name="poll-reviews" + name="poll-reviews", ), ] diff --git a/review/views.py b/review/views.py index da438ae..2e354ab 100644 --- a/review/views.py +++ b/review/views.py @@ -1,6 +1,5 @@ from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_schema -from rest_framework import status, viewsets -from rest_framework.decorators import action +from rest_framework import viewsets from rest_framework.response import Response from .serializers import ReviewSerializer @@ -43,7 +42,11 @@ def retrieve(self, request, pk=None): request=None, parameters=[ OpenApiParameter( - name="id", type=int, required=True, location="path", description="App ID to fetch reviews for" + name="id", + type=int, + required=True, + location="path", + description="App ID to fetch reviews for", ), OpenApiParameter( name="date_from", type=str, description="Start date (YYYY-MM-DD)", required=False From 88adedc99ec05adcfdaf187c51cbebdeb8597420 Mon Sep 17 00:00:00 2001 From: Anyerrr Date: Wed, 30 Apr 2025 11:57:04 +0200 Subject: [PATCH 30/42] =?UTF-8?q?feat(api-rest):=20Crear=20les=20views=20i?= =?UTF-8?q?=20urls=20de=20l=E2=80=99app=20metrics=20i=20documentar-ho?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs: #64 --- config/urls.py | 1 + metric/repositories.py | 53 +++++++ metric/repositories/__init__.py | 0 metric/repositories/metric_repository.py | 7 - metric/serializers.py | 27 ++++ metric/services.py | 47 ++++++ metric/services/__init__.py | 0 .../services/metric_registration_service.py | 22 --- metric/urls.py | 12 ++ metric/views.py | 134 ++++++++++++++++++ source/repositories.py | 10 ++ source/serializers.py | 6 + source/services.py | 6 + source/views.py | 22 +-- .../unit/test_metric_registration_service.py | 24 ---- 15 files changed, 307 insertions(+), 64 deletions(-) create mode 100644 metric/repositories.py delete mode 100644 metric/repositories/__init__.py delete mode 100644 metric/repositories/metric_repository.py create mode 100644 metric/services.py delete mode 100644 metric/services/__init__.py delete mode 100644 metric/services/metric_registration_service.py create mode 100644 metric/urls.py delete mode 100644 tests/unit/test_metric_registration_service.py diff --git a/config/urls.py b/config/urls.py index 9aed731..60775e6 100644 --- a/config/urls.py +++ b/config/urls.py @@ -33,4 +33,5 @@ def home(request): path("api/", include("app.urls")), path("api/", include("source.urls")), path("api/", include("review.urls")), + path("api/", include("metric.urls")), ] diff --git a/metric/repositories.py b/metric/repositories.py new file mode 100644 index 0000000..3c5e8dd --- /dev/null +++ b/metric/repositories.py @@ -0,0 +1,53 @@ +from django.shortcuts import get_object_or_404 + +from .models import Metric, MetricValue + + +class MetricRepository: + def get_all(self): + return Metric.objects.all() + + def get_by_id(self, pk): + return get_object_or_404(Metric, id=pk) + + def create(self, data): + return Metric.objects.create(**data) + + def update(self, instance, data): + for attr, value in data.items(): + setattr(instance, attr, value) + instance.save() + return instance + + def delete(self, instance): + instance.delete() + + def add_sources(self, instance, sources_ids): + instance.sources.add(*sources_ids) + instance.save() + return instance + + def remove_sources(self, instance, sources_ids): + instance.sources.remove(*sources_ids) + instance.save() + return instance + + +class MetricValueRepository: + def get_all(self): + return MetricValue.objects.all() + + def get_by_id(self, pk): + return get_object_or_404(MetricValue, id=pk) + + def create(self, data): + return MetricValue.objects.create(**data) + + def update(self, instance, data): + for attr, value in data.items(): + setattr(instance, attr, value) + instance.save() + return instance + + def delete(self, instance): + instance.delete() diff --git a/metric/repositories/__init__.py b/metric/repositories/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/metric/repositories/metric_repository.py b/metric/repositories/metric_repository.py deleted file mode 100644 index dec911a..0000000 --- a/metric/repositories/metric_repository.py +++ /dev/null @@ -1,7 +0,0 @@ -from metric.models import Metric - - -class MetricRepository: - @staticmethod - def get_or_create(code, defaults): - return Metric.objects.get_or_create(code=code, defaults=defaults) diff --git a/metric/serializers.py b/metric/serializers.py index e07a3b5..5e21823 100644 --- a/metric/serializers.py +++ b/metric/serializers.py @@ -1,5 +1,32 @@ from rest_framework import serializers +from .constants.value_types import MetricValueType +from .models import Metric, MetricValue + + +class MetricSerializer(serializers.ModelSerializer): + value_type = serializers.ChoiceField(choices=MetricValueType.choices) + + class Meta: + model = Metric + fields = ["id", "code", "name", "description", "value_type", "sources"] + read_only_fields = ["sources"] + + +class MetricValueSerializer(serializers.ModelSerializer): + class Meta: + model = MetricValue + fields = [ + "id", + "app", + "metric", + "source", + "value", + "retrieved_at", + ] + + read_only_fields = fields + class LinkMetricsSerializer(serializers.Serializer): metrics = serializers.ListField( diff --git a/metric/services.py b/metric/services.py new file mode 100644 index 0000000..c7dda3b --- /dev/null +++ b/metric/services.py @@ -0,0 +1,47 @@ +from .repositories import MetricRepository, MetricValueRepository + + +class MetricService: + def __init__(self): + self.repo = MetricRepository() + + def list_metrics(self): + return self.repo.get_all() + + def get_metric(self, pk): + return self.repo.get_by_id(pk) + + def create_metric(self, validated_data): + return self.repo.create(validated_data) + + def update_metric(self, instance, validated_data): + return self.repo.update(instance, validated_data) + + def delete_metric(self, instance): + return self.repo.delete(instance) + + def add_sources(self, instance, sources_ids): + return self.repo.add_sources(instance, sources_ids) + + def remove_sources(self, instance, sources_ids): + return self.repo.remove_sources(instance, sources_ids) + + +class MetricValueService: + def __init__(self): + self.repo = MetricValueRepository() + + def list_metric_values(self): + return self.repo.get_all() + + def get_metric_value(self, pk): + return self.repo.get_by_id(pk) + + def create_metric_value(self, validated_data): + return self.repo.create(validated_data) + + def update_metric_value(self, instance, validated_data): + return self.repo.update(instance, validated_data) + + def delete_metric_value(self, instance): + return self.repo.delete(instance) diff --git a/metric/services/__init__.py b/metric/services/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/metric/services/metric_registration_service.py b/metric/services/metric_registration_service.py deleted file mode 100644 index fd979c7..0000000 --- a/metric/services/metric_registration_service.py +++ /dev/null @@ -1,22 +0,0 @@ -from metric.definitions.metrics import METRICS -from metric.repositories.metric_repository import MetricRepository - - -class MetricRegistrationService: - def __init__(self, repository=None): - self.repository = repository or MetricRepository() - - def register_all(self): - for m in METRICS: - metric, created = self.repository.get_or_create( - code=m["code"], - defaults={ - "name": m["name"], - "description": m["description"], - "value_type": m["value_type"].value, - }, - ) - if created: - print(f"✅ Mètrica registrada: {m['code']}") - else: - print(f"ℹ️ Ja existia: {m['code']}") diff --git a/metric/urls.py b/metric/urls.py new file mode 100644 index 0000000..9891c6c --- /dev/null +++ b/metric/urls.py @@ -0,0 +1,12 @@ +from django.urls import include, path +from rest_framework.routers import DefaultRouter + +from metric.views import MetricValueViewSet, MetricViewSet + +router = DefaultRouter() +router.register(r"metrics", MetricViewSet, basename="metric") +router.register(r"metric-values", MetricValueViewSet, basename="metric_value") + +urlpatterns = [ + path("", include(router.urls)), +] diff --git a/metric/views.py b/metric/views.py index e69de29..865a38d 100644 --- a/metric/views.py +++ b/metric/views.py @@ -0,0 +1,134 @@ +from drf_spectacular.utils import OpenApiParameter, extend_schema +from rest_framework import status, viewsets +from rest_framework.decorators import action +from rest_framework.response import Response + +from source.serializers import LinkSourceSerializer + +from .serializers import MetricSerializer, MetricValueSerializer +from .services import MetricService, MetricValueService + + +class MetricViewSet(viewsets.ViewSet): + service = MetricService() + + @extend_schema( + responses=MetricSerializer(many=True), + tags=["Metrics"], + ) + def list(self, request): + metrics = self.service.list_metrics() + serializer = MetricSerializer(metrics, many=True) + return Response(serializer.data) + + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses=MetricSerializer, + tags=["Metrics"], + ) + def retrieve(self, request, pk=None): + metric = self.service.get_metric(pk) + serializer = MetricSerializer(metric) + return Response(serializer.data) + + @extend_schema( + request=MetricSerializer, + responses=MetricSerializer, + description=( + "Creates a new data metric.\n\n" + "The 'source' field is read-only and cannot be set directly on creation." + ), + tags=["Metrics"], + ) + def create(self, request): + serializer = MetricSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + metric = self.service.create_metric(serializer.validated_data) + return Response(MetricSerializer(metric).data, status=status.HTTP_201_CREATED) + + @extend_schema( + request=MetricSerializer, + responses=MetricSerializer, + tags=["Metrics"], + ) + def update(self, request, pk=None): + metric = self.service.get_metric(pk) + serializer = MetricSerializer(metric, data=request.data) + serializer.is_valid(raise_exception=True) + updated_metric = self.service.update_metric(metric, serializer.validated_data) + return Response(MetricSerializer(updated_metric).data) + + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses={204: None}, + tags=["Metrics"], + ) + def destroy(self, request, pk=None): + metric = self.service.get_metric(pk) + self.service.delete_metric(metric) + return Response(status=status.HTTP_204_NO_CONTENT) + + @extend_schema( + request=LinkSourceSerializer, + responses=MetricSerializer, + description="Adds a list of source IDs to an existing " + "metric without removing existing ones.", + tags=["Metrics"], + ) + @action(detail=True, methods=["post"], url_path="sources") + def add_metrics(self, request, pk=None): + sources_ids = request.data.get("metrics", []) + + if not isinstance(sources_ids, list): + return Response( + {"error": "metrics must be a list of IDs"}, status=status.HTTP_400_BAD_REQUEST + ) + + metric = self.service.get_metric(pk) + metric = self.service.add_sources(metric, sources_ids) + + return Response(MetricSerializer(metric).data, status=status.HTTP_200_OK) + + @extend_schema( + request=LinkSourceSerializer, + responses=MetricSerializer, + description="Removes a list of source IDs from an existing metric.", + tags=["Metrics"], + methods=["DELETE"], + ) + @action(detail=True, methods=["delete"], url_path="sources") + def remove_metrics(self, request, pk=None): + sources_ids = request.data.get("sources", []) + + if not isinstance(sources_ids, list): + return Response( + {"error": "metrics must be a list of IDs"}, status=status.HTTP_400_BAD_REQUEST + ) + + metric = self.service.get_metric(pk) + metric = self.service.remove_sources(metric, sources_ids) + + return Response(MetricSerializer(metric).data, status=status.HTTP_200_OK) + + +class MetricValueViewSet(viewsets.ViewSet): + service = MetricValueService() + + @extend_schema( + responses=MetricValueSerializer(many=True), + tags=["Metrics"], + ) + def list(self, request): + metrics = self.service.list_metric_values() + serializer = MetricValueSerializer(metrics, many=True) + return Response(serializer.data) + + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses=MetricValueSerializer, + tags=["Metrics"], + ) + def retrieve(self, request, pk=None): + metric = self.service.get_metric_value(pk) + serializer = MetricValueSerializer(metric) + return Response(serializer.data) diff --git a/source/repositories.py b/source/repositories.py index 7f5b9f1..b205643 100644 --- a/source/repositories.py +++ b/source/repositories.py @@ -21,3 +21,13 @@ def update(self, instance, data): def delete(self, instance): instance.delete() + + def add_metrics(self, instance, metrics_ids): + instance.metrics.add(*metrics_ids) + instance.save() + return instance + + def remove_metrics(self, instance, metrics_ids): + instance.metrics.remove(*metrics_ids) + instance.save() + return instance diff --git a/source/serializers.py b/source/serializers.py index d2a1610..10814a2 100644 --- a/source/serializers.py +++ b/source/serializers.py @@ -26,3 +26,9 @@ class Meta: "metrics", ] read_only_fields = ["metrics"] + + +class LinkSourceSerializer(serializers.Serializer): + sources = serializers.ListField( + child=serializers.IntegerField(), help_text="List of source IDs to link to the metric." + ) diff --git a/source/services.py b/source/services.py index 59685f8..cebba0c 100644 --- a/source/services.py +++ b/source/services.py @@ -30,6 +30,12 @@ def update_source(self, instance, validated_data): def delete_source(self, instance): return self.repo.delete(instance) + def add_metrics(self, instance, metrics_ids): + return self.repo.add_metrics(instance, metrics_ids) + + def remove_metrics(self, instance, metrics_ids): + return self.repo.remove_metrics(instance, metrics_ids) + @staticmethod def load_sources(): return [cls() for cls in SourceAdapter.__subclasses__()] diff --git a/source/views.py b/source/views.py index 1817e64..5672410 100644 --- a/source/views.py +++ b/source/views.py @@ -43,7 +43,7 @@ def retrieve(self, request, pk=None): def create(self, request): serializer = SourceSerializer(data=request.data) serializer.is_valid(raise_exception=True) - source = serializer.save() + source = self.service.create_source(serializer.validated_data) return Response(SourceSerializer(source).data, status=status.HTTP_201_CREATED) @extend_schema( @@ -55,7 +55,7 @@ def update(self, request, pk=None): source = self.service.get_source(pk) serializer = SourceSerializer(source, data=request.data) serializer.is_valid(raise_exception=True) - updated_source = serializer.save() + updated_source = self.service.update_source(source, serializer.validated_data) return Response(SourceSerializer(updated_source).data) @extend_schema( @@ -65,19 +65,19 @@ def update(self, request, pk=None): ) def destroy(self, request, pk=None): source = self.service.get_source(pk) - source.delete() + source = self.service.delete_source(source) return Response(status=status.HTTP_204_NO_CONTENT) @extend_schema( request=LinkMetricsSerializer, + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], responses=SourceSerializer, description="Adds a list of metric IDs to an existing " "source without removing existing ones.", tags=["Sources"], ) - @action(detail=True, methods=["post"], url_path="add-metrics") + @action(detail=True, methods=["post"], url_path="metrics") def add_metrics(self, request, pk=None): - source = self.service.get_source(pk) metrics_ids = request.data.get("metrics", []) if not isinstance(metrics_ids, list): @@ -85,20 +85,20 @@ def add_metrics(self, request, pk=None): {"error": "metrics must be a list of IDs"}, status=status.HTTP_400_BAD_REQUEST ) - source.metrics.add(*metrics_ids) - source.save() + source = self.service.get_source(pk) + source = self.service.add_metrics(source, metrics_ids) return Response(SourceSerializer(source).data, status=status.HTTP_200_OK) @extend_schema( request=LinkMetricsSerializer, + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], responses=SourceSerializer, description="Removes a list of metric IDs from an existing source.", tags=["Sources"], ) - @action(detail=True, methods=["post"], url_path="remove-metrics") + @action(detail=True, methods=["delete"], url_path="metrics") def remove_metrics(self, request, pk=None): - source = self.service.get_source(pk) metrics_ids = request.data.get("metrics", []) if not isinstance(metrics_ids, list): @@ -106,7 +106,7 @@ def remove_metrics(self, request, pk=None): {"error": "metrics must be a list of IDs"}, status=status.HTTP_400_BAD_REQUEST ) - source.metrics.remove(*metrics_ids) - source.save() + source = self.service.get_source(pk) + source = self.service.remove_metrics(source, metrics_ids) return Response(SourceSerializer(source).data, status=status.HTTP_200_OK) diff --git a/tests/unit/test_metric_registration_service.py b/tests/unit/test_metric_registration_service.py deleted file mode 100644 index 39d4da9..0000000 --- a/tests/unit/test_metric_registration_service.py +++ /dev/null @@ -1,24 +0,0 @@ -from metric.definitions.metrics import METRICS -from metric.services.metric_registration_service import MetricRegistrationService - - -class FakeMetricRepository: - def __init__(self): - self.created = [] - - def get_or_create(self, code, defaults): - # Simula que totes les mètriques són noves - self.created.append((code, defaults)) - return {"code": code, **defaults}, True - - -def test_metric_registration_service_registers_all_metrics(): - fake_repo = FakeMetricRepository() - service = MetricRegistrationService(repository=fake_repo) - - service.register_all() - - assert len(fake_repo.created) == len(METRICS) - - codes = [c for c, _ in fake_repo.created] - assert "average_rating" in codes From 1173d55570af2abd461137d076e80af3d1964ff0 Mon Sep 17 00:00:00 2001 From: Anyer Date: Fri, 2 May 2025 08:10:55 +0200 Subject: [PATCH 31/42] =?UTF-8?q?feature(api-rest):=20Correci=C3=B3=20d'er?= =?UTF-8?q?rors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs # --- metric/definitions/__init__.py | 0 metric/definitions/metrics.py | 30 -------------------------- source/adapters/google_play_scraper.py | 15 +++++++------ tests/integration/test_end_to_end.py | 5 ++++- 4 files changed, 12 insertions(+), 38 deletions(-) delete mode 100644 metric/definitions/__init__.py delete mode 100644 metric/definitions/metrics.py diff --git a/metric/definitions/__init__.py b/metric/definitions/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/metric/definitions/metrics.py b/metric/definitions/metrics.py deleted file mode 100644 index a111d9f..0000000 --- a/metric/definitions/metrics.py +++ /dev/null @@ -1,30 +0,0 @@ -from metric.constants import MetricCodes -from metric.constants.value_types import MetricValueType - -METRICS = [ - { - "code": MetricCodes.AVERAGE_RATING, - "name": "Average Rating", - "description": "Valoració mitjana de l'app", - "value_type": MetricValueType.FLOAT, - }, - { - "code": MetricCodes.TOTAL_REVIEWS, - "name": "Total Reviews", - "description": "Nombre total de ressenyes", - "value_type": MetricValueType.INTEGER, - }, - { - "code": MetricCodes.DAILY_NEWS_BLOG_MENTIONS, - "name": "Daily News Blog Mentions", - "description": "Nombre de mencions diàries a blogs de notícies", - "value_type": MetricValueType.INTEGER, - }, - { - "code": MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS, - "name": "Daily Reddit Mentions", - "description": "Nombre de mencions diàries a Reddit", - "value_type": MetricValueType.INTEGER, - }, - # Afegir més mètriques aquí -] diff --git a/source/adapters/google_play_scraper.py b/source/adapters/google_play_scraper.py index a377b3f..039d617 100644 --- a/source/adapters/google_play_scraper.py +++ b/source/adapters/google_play_scraper.py @@ -38,13 +38,14 @@ def fetch(self, app_id: int, metric: str): return None result = self.lookup_app(package_name) - - return { - MetricCodes.AVERAGE_RATING: result.get("score"), - MetricCodes.TOTAL_REVIEWS: result.get("reviews"), - MetricCodes.TOTAL_DOWNLOADS: result.get("realInstalls"), - MetricCodes.LAST_UPDATE_DATE: result.get("lastUpdatedOn"), - }.get(metric) + if result: + return { + MetricCodes.AVERAGE_RATING: result.get("score"), + MetricCodes.TOTAL_REVIEWS: result.get("reviews"), + MetricCodes.TOTAL_DOWNLOADS: result.get("realInstalls"), + MetricCodes.LAST_UPDATE_DATE: result.get("lastUpdatedOn"), + }.get(metric) + return None @staticmethod def lookup_app(package_name: str) -> dict | None: diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py index 898c011..c25906e 100644 --- a/tests/integration/test_end_to_end.py +++ b/tests/integration/test_end_to_end.py @@ -27,9 +27,12 @@ def test_full_flow_with_real_adapter(): # Adapter real (ja ha de formar part del projecte i registrar-se automàticament) adapters = SourceService().load_sources() + itunes_adapter = next((a for a in adapters if a.code == "itunes"), None) + + assert itunes_adapter is not None, "iTunes adapter not found in loaded sources" strategy = GenericMetricStrategy("average_rating") - values = strategy.compute_all(app.id, adapters) + values = strategy.compute_all(app.id, [itunes_adapter]) for v in values: print(f"metric={v.metric.code}, app={v.app.code}, source={v.source.code}, value={v.value}") From a174b3932e700744a38d0c43afdb5aebeebecc92 Mon Sep 17 00:00:00 2001 From: Anyer Date: Sun, 4 May 2025 20:50:51 +0200 Subject: [PATCH 32/42] =?UTF-8?q?feature(metrics):=20Implementar=20les=20m?= =?UTF-8?q?=C3=A8triques=20restants=20al=20sistema=20(models,=20processame?= =?UTF-8?q?nt=20i=20emmagatzematge)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #76 --- app/repositories.py | 15 ++- app/services.py | 23 +++- metric/management/commands/poll_metrics.py | 111 +++++++++------ .../0004_alter_metricvalue_source.py | 25 ++++ metric/migrations/0005_add_is_internal.py | 16 +++ metric/models.py | 8 +- metric/repositories.py | 36 +++-- source/adapters/base.py | 8 +- source/adapters/google_play_scraper.py | 77 +++++------ source/adapters/itunes.py | 55 ++++---- source/adapters/news.py | 45 ++++--- source/adapters/reddit.py | 46 ++++--- source/repositories.py | 26 +++- source/services.py | 25 ++-- tests/adapters/__init__.py | 0 tests/adapters/test_all_adapters.py | 27 ++++ tests/adapters/test_google_play_scraper.py | 127 ++++++++++++++++++ tests/adapters/test_itunes_adapter.py | 26 ++++ tests/adapters/test_news.py | 31 +++++ tests/adapters/test_reddit.py | 31 +++++ tests/conftest.py | 43 ++++++ .../unit/test_google_play_scraper_adapter.py | 27 ---- tests/unit/test_itunes_adapter.py | 21 --- tests/unit/test_news_adapter.py | 30 ----- tests/unit/test_reddit_adapter.py | 34 ----- 25 files changed, 608 insertions(+), 305 deletions(-) create mode 100644 metric/migrations/0004_alter_metricvalue_source.py create mode 100644 metric/migrations/0005_add_is_internal.py create mode 100644 tests/adapters/__init__.py create mode 100644 tests/adapters/test_all_adapters.py create mode 100644 tests/adapters/test_google_play_scraper.py create mode 100644 tests/adapters/test_itunes_adapter.py create mode 100644 tests/adapters/test_news.py create mode 100644 tests/adapters/test_reddit.py create mode 100644 tests/conftest.py delete mode 100644 tests/unit/test_google_play_scraper_adapter.py delete mode 100644 tests/unit/test_itunes_adapter.py delete mode 100644 tests/unit/test_news_adapter.py delete mode 100644 tests/unit/test_reddit_adapter.py diff --git a/app/repositories.py b/app/repositories.py index 09897cd..1b47aae 100644 --- a/app/repositories.py +++ b/app/repositories.py @@ -4,20 +4,25 @@ class AppRepository: - def get_all(self): + @staticmethod + def get_all(): return App.objects.all() - def get_by_id(self, app_id): + @staticmethod + def get_by_id(app_id): return get_object_or_404(App, id=app_id) - def create(self, data): + @staticmethod + def create(data): return App.objects.create(**data) - def update(self, instance, data): + @staticmethod + def update(instance, data): for attr, value in data.items(): setattr(instance, attr, value) instance.save() return instance - def delete(self, instance): + @staticmethod + def delete(instance): instance.delete() diff --git a/app/services.py b/app/services.py index 47b4fc9..02a035b 100644 --- a/app/services.py +++ b/app/services.py @@ -1,16 +1,29 @@ from datetime import datetime -from source.adapters.google_play_scraper import GooglePlayScraperAdapter -from source.adapters.itunes import ItunesSearchAPIAdapter - from .repositories import AppRepository class AppService: def __init__(self, itunes_adapter=None, google_play_adapter=None): self.repo = AppRepository() - self.itunes_adapter = itunes_adapter or ItunesSearchAPIAdapter() - self.google_play_adapter = google_play_adapter or GooglePlayScraperAdapter() + self._itunes_adapter = itunes_adapter + self._google_play_adapter = google_play_adapter + + @property + def itunes_adapter(self): + if self._itunes_adapter is None: + from source.adapters.itunes import ItunesSearchAPIAdapter + + self._itunes_adapter = ItunesSearchAPIAdapter() + return self._itunes_adapter + + @property + def google_play_adapter(self): + if self._google_play_adapter is None: + from source.adapters.google_play_scraper import GooglePlayScraperAdapter + + self._google_play_adapter = GooglePlayScraperAdapter() + return self._google_play_adapter def list_apps(self): return self.repo.get_all() diff --git a/metric/management/commands/poll_metrics.py b/metric/management/commands/poll_metrics.py index 6a9e943..5391166 100644 --- a/metric/management/commands/poll_metrics.py +++ b/metric/management/commands/poll_metrics.py @@ -1,10 +1,12 @@ +from datetime import datetime + from django.core.management.base import BaseCommand -from django.utils import timezone +from django.utils.timezone import now from app.models import App from metric.constants import MetricCodes -from metric.models import Metric -from metric.strategies.generic import GenericMetricStrategy +from metric.models import Metric, MetricValue +from source.models import Source from source.services import SourceService @@ -40,41 +42,68 @@ class Command(BaseCommand): help = "Recull mètriques per a totes les apps i fonts disponibles" def handle(self, *args, **options): - print("[🔁] Iniciant recollida de mètriques diàries...") - - if validate_metric_codes(): - adapters = SourceService().load_sources() - apps = App.objects.all() - metrics = Metric.objects.prefetch_related("sources").all() - - for metric in metrics: - print(f"📌 Processant mètrica: {metric.code}") - strategy = GenericMetricStrategy(metric.code) - source_codes = metric.sources.values_list("code", flat=True) - matching_adapters = [a for a in adapters if a.code in source_codes] - - for app in apps: - print(f" 🔍 App: {app.code}") - values = strategy.compute_all(app.id, matching_adapters) - - for value in values: - if value.value is None: - print(f" ⚠️ {metric.code} ({value.source}) no ha retornat valor.") - continue - - exists = value.__class__.objects.filter( - app=app, - metric=metric, - source=value.source, - retrieved_at__date=timezone.now().date(), - ).exists() - - if not exists: - value.save() - print(f" ✅ Valor guardat: {value.value} des de {value.source}") - else: - print( - f" ℹ️ Ja existia: {metric.code} per {app.code}" - f" via {value.source}, nou valor: {value.value}" - ) - print("✅ Procés completat.") + print("[🔁] Iniciant recollida de mètriques...") + + # 🔃 Carrega dades necessàries + adapters = SourceService().load_sources() + apps = App.objects.all() + metrics = list(Metric.objects.prefetch_related("sources").all()) + + # 🔀 Separa mètriques + external_metrics = [m for m in metrics if not m.is_internal] + internal_metrics = [m for m in metrics if m.is_internal] + + # ⚡ Caches + metric_cache = {m.code: m for m in metrics} + source_cache = {a.code: Source.objects.get(code=a.code) for a in adapters} + + # 🔁 EXTERNAL: Per adapters + for app in apps: + print(f"\n🔍 App: {app.code}") + for adapter in adapters: + print("adapter actual: ", adapter.code) + source_code = adapter.code + available_metrics = [ + m.code + for m in external_metrics + if source_code in m.sources.values_list("code", flat=True) + ] + if not available_metrics: + continue + + try: + result = adapter.fetch(app.id, available_metrics) + except Exception as e: + print(f"❌ Error en adapter `{source_code}`: {e}") + continue + + for metric_code, value in result.items(): + if value in [None, ""]: + print(f" ⚠️ {metric_code} no ha retornat valor.") + continue + + metric = metric_cache[metric_code] + source = source_cache[source_code] + + already_exists = MetricValue.objects.filter( + app=app, metric=metric, source=source, retrieved_at__date=now().date() + ).exists() + + if already_exists: + print(f" ℹ️ Ja existia: {metric_code} — {value}") + continue + + MetricValue.objects.create( + app=app, + metric=metric, + source=source, + value=value, + retrieved_at=datetime.now(), + ) + print(f" ✅ Guardat {metric_code}: {value}") + + # 🔁 INTERNAL: opcional (implementa si tens estratègies internes) + if internal_metrics: + print("\n📦 Tractament de mètriques internes no implementat (pendents).") + + print("\n✅ Procés de recollida completat.") diff --git a/metric/migrations/0004_alter_metricvalue_source.py b/metric/migrations/0004_alter_metricvalue_source.py new file mode 100644 index 0000000..f6e8e26 --- /dev/null +++ b/metric/migrations/0004_alter_metricvalue_source.py @@ -0,0 +1,25 @@ +# Generated by Django 5.1.7 on 2025-05-04 17:49 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("metric", "0003_alter_metric_code"), + ("source", "0003_source_metrics"), + ] + + operations = [ + migrations.AlterField( + model_name="metricvalue", + name="source", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="source.source", + ), + ), + ] diff --git a/metric/migrations/0005_add_is_internal.py b/metric/migrations/0005_add_is_internal.py new file mode 100644 index 0000000..ee3b007 --- /dev/null +++ b/metric/migrations/0005_add_is_internal.py @@ -0,0 +1,16 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("metric", "0004_alter_metricvalue_source"), + ] + + operations = [ + migrations.AddField( + model_name="metric", + name="is_internal", + field=models.BooleanField(default=False), + ), + ] diff --git a/metric/models.py b/metric/models.py index 602c5b5..2c6bb8d 100644 --- a/metric/models.py +++ b/metric/models.py @@ -9,6 +9,7 @@ class Metric(models.Model): name = models.CharField(max_length=100) description = models.TextField() value_type = models.CharField(max_length=10, choices=MetricValueType.choices) + is_internal = models.BooleanField(default=False) def __str__(self): return self.name @@ -17,9 +18,12 @@ def __str__(self): class MetricValue(models.Model): app = models.ForeignKey(App, on_delete=models.CASCADE) metric = models.ForeignKey(Metric, on_delete=models.CASCADE) - source = models.ForeignKey("source.Source", on_delete=models.CASCADE) + source = models.ForeignKey("source.Source", on_delete=models.CASCADE, null=True, blank=True) value = models.CharField(max_length=255) retrieved_at = models.DateTimeField(auto_now_add=True) def __str__(self): - return f"{self.metric.name} - {self.source.name} - {self.app.name}" + return ( + f"{self.metric.name} - {self.source.name if self.source else 'Internal'} " + f"- {self.app.name}" + ) diff --git a/metric/repositories.py b/metric/repositories.py index 3c5e8dd..a132a18 100644 --- a/metric/repositories.py +++ b/metric/repositories.py @@ -4,50 +4,62 @@ class MetricRepository: - def get_all(self): + @staticmethod + def get_all(): return Metric.objects.all() - def get_by_id(self, pk): + @staticmethod + def get_by_id(pk): return get_object_or_404(Metric, id=pk) - def create(self, data): + @staticmethod + def create(data): return Metric.objects.create(**data) - def update(self, instance, data): + @staticmethod + def update(instance, data): for attr, value in data.items(): setattr(instance, attr, value) instance.save() return instance - def delete(self, instance): + @staticmethod + def delete(instance): instance.delete() - def add_sources(self, instance, sources_ids): + @staticmethod + def add_sources(instance, sources_ids): instance.sources.add(*sources_ids) instance.save() return instance - def remove_sources(self, instance, sources_ids): + @staticmethod + def remove_sources(instance, sources_ids): instance.sources.remove(*sources_ids) instance.save() return instance class MetricValueRepository: - def get_all(self): + @staticmethod + def get_all(): return MetricValue.objects.all() - def get_by_id(self, pk): + @staticmethod + def get_by_id(pk): return get_object_or_404(MetricValue, id=pk) - def create(self, data): + @staticmethod + def create(data): return MetricValue.objects.create(**data) - def update(self, instance, data): + @staticmethod + def update(instance, data): for attr, value in data.items(): setattr(instance, attr, value) instance.save() return instance - def delete(self, instance): + @staticmethod + def delete(instance): instance.delete() diff --git a/source/adapters/base.py b/source/adapters/base.py index ca4a01e..52d27b8 100644 --- a/source/adapters/base.py +++ b/source/adapters/base.py @@ -8,12 +8,16 @@ class SourceAdapter(ABC): code: str name: str type: SourceType - url: str | None + url: Optional[str] supported_metrics: list[str] api_key: Optional[str] = None + def __init__(self): + if not hasattr(self, "code") or not self.code: + raise NotImplementedError("Adapter must define 'code' and load source config.") + @abstractmethod def supports_metric(self, metric: str) -> bool: ... @abstractmethod - def fetch(self, app_id: int, metric: str) -> str | None: ... + def fetch(self, app_id: int, metrics: list[str]) -> dict[str, str]: ... diff --git a/source/adapters/google_play_scraper.py b/source/adapters/google_play_scraper.py index 039d617..981d37c 100644 --- a/source/adapters/google_play_scraper.py +++ b/source/adapters/google_play_scraper.py @@ -3,50 +3,28 @@ from google_play_scraper import app as gp_app from google_play_scraper import reviews -from rest_framework.exceptions import NotFound, ParseError, ValidationError +from rest_framework.exceptions import ParseError, ValidationError -from app.models import App +from app.services import AppService from metric.constants import MetricCodes from source.adapters.base import SourceAdapter -from source.constants.source_type import SourceType +from source.services import SourceService class GooglePlayScraperAdapter(SourceAdapter): code = "google_play" - name = "Google Play Scraper" - type = SourceType.SCRAPER - url = "https://play.google.com" - supported_metrics = [ - MetricCodes.AVERAGE_RATING, - MetricCodes.TOTAL_REVIEWS, - MetricCodes.TOTAL_DOWNLOADS, - MetricCodes.LAST_UPDATE_DATE, - ] + + def __init__(self): + super().__init__() + source_data = SourceService().get_source_data(code=self.code) + self.name = source_data["name"] + self.type = source_data["type"] + self.url = source_data["url"] + self.supported_metrics = source_data["supported_metrics"] def supports_metric(self, metric: str): return metric in self.supported_metrics - def fetch(self, app_id: int, metric: str): - try: - app_instance = App.objects.get(id=app_id) - package_name = app_instance.playstore_id - except App.DoesNotExist: - print(f"[GooglePlayScraperAdapter] App with id {app_id} does not exist.") - return None - except AttributeError: - print("[GooglePlayScraperAdapter] App is missing 'playstore_id' field.") - return None - - result = self.lookup_app(package_name) - if result: - return { - MetricCodes.AVERAGE_RATING: result.get("score"), - MetricCodes.TOTAL_REVIEWS: result.get("reviews"), - MetricCodes.TOTAL_DOWNLOADS: result.get("realInstalls"), - MetricCodes.LAST_UPDATE_DATE: result.get("lastUpdatedOn"), - }.get(metric) - return None - @staticmethod def lookup_app(package_name: str) -> dict | None: try: @@ -56,17 +34,28 @@ def lookup_app(package_name: str) -> dict | None: print(f"[GooglePlayScraperAdapter] Error fetching app data: {e}") return None - def fetch_reviews(self, app_id: int, date_from=None, date_to=None): - try: - app_instance = App.objects.get(id=app_id) - package_name = app_instance.playstore_id - except App.DoesNotExist: - raise NotFound(f"App with id {app_id} does not exist.") - except AttributeError: - raise ParseError("App is missing 'playstore_id' field.") + def fetch(self, app_id: int, metrics: list[str]): + app = AppService().get_app(app_id=app_id) + if not app: + return {} + + result = self.lookup_app(app.playstore_id) - if not package_name: - raise ParseError("App has no 'playstore_id' set.") + metric_map = { + MetricCodes.AVERAGE_RATING: str(result.get("score") or ""), + MetricCodes.TOTAL_REVIEWS: str(result.get("reviews") or ""), + MetricCodes.TOTAL_DOWNLOADS: str(result.get("realInstalls") or ""), + MetricCodes.LAST_UPDATE_DATE: str(result.get("lastUpdatedOn") or ""), + } + + return { + metric: metric_map.get(metric, "") for metric in metrics if self.supports_metric(metric) + } + + def fetch_reviews(self, app_id: int, date_from=None, date_to=None): + app = AppService().get_app(app_id=app_id) + if not app: + return [] # Dates per defecte: ahir yesterday = date.today() - timedelta(days=1) @@ -88,7 +77,7 @@ def fetch_reviews(self, app_id: int, date_from=None, date_to=None): while True: batch, next_token = reviews( - package_name, + app.playstore_id, lang="en", country="us", count=count_per_request, diff --git a/source/adapters/itunes.py b/source/adapters/itunes.py index 493958e..7226016 100644 --- a/source/adapters/itunes.py +++ b/source/adapters/itunes.py @@ -1,45 +1,48 @@ import requests -from app.models import App +from app.services import AppService from metric.constants import MetricCodes from source.adapters.base import SourceAdapter -from source.constants.source_type import SourceType +from source.services import SourceService class ItunesSearchAPIAdapter(SourceAdapter): code = "itunes" - name = "App Store" - type = SourceType.API - url = "https://itunes.apple.com" - supported_metrics = [MetricCodes.AVERAGE_RATING, MetricCodes.TOTAL_REVIEWS] - def supports_metric(self, metric: str) -> bool: - return metric in self.supported_metrics - - def fetch(self, app_id: int, metric: str): - try: - app = App.objects.get(id=app_id) - except App.DoesNotExist: - return None - - result = self.lookup_app(app.appstore_id) - if not result: - return None + def __init__(self): + super().__init__() + source_data = SourceService().get_source_data(code=self.code) + self.name = source_data["name"] + self.type = source_data["type"] + self.url = source_data["url"] + self.supported_metrics = source_data["supported_metrics"] - return { - MetricCodes.AVERAGE_RATING: result.get("averageUserRating"), - MetricCodes.TOTAL_REVIEWS: result.get("userRatingCount"), - }.get(metric) + def supports_metric(self, metric: str): + return metric in self.supported_metrics - def lookup_app(self, appstore_id: str) -> dict | None: + def lookup_app(self, appstore_id: str): response = requests.get(f"{self.url}/lookup?id={appstore_id}") if not response.ok: return None data = response.json() results = data.get("results", []) + return results[0] if results else None - if not results: - return None + def fetch(self, app_id: int, metrics: list[str]): + app = AppService().get_app(app_id=app_id) + if not app: + return {} - return results[0] + result = self.lookup_app(app.appstore_id) + if not result: + return {} + + metric_map = { + MetricCodes.AVERAGE_RATING: str(result.get("averageUserRating") or ""), + MetricCodes.TOTAL_REVIEWS: str(result.get("userRatingCount") or ""), + } + + return { + metric: metric_map.get(metric, "") for metric in metrics if self.supports_metric(metric) + } diff --git a/source/adapters/news.py b/source/adapters/news.py index 727f270..1bdea50 100644 --- a/source/adapters/news.py +++ b/source/adapters/news.py @@ -1,53 +1,58 @@ import os -from datetime import datetime +from datetime import datetime, timedelta import requests -from app.models import App +from app.services import AppService from metric.constants import MetricCodes from source.adapters.base import SourceAdapter -from source.constants.source_type import SourceType +from source.services import SourceService class NewsAPIAdapter(SourceAdapter): code = "news" - name = "News API" - type = SourceType.API - url = "https://newsapi.org/v2" - supported_metrics = [MetricCodes.DAILY_NEWS_BLOG_MENTIONS] def __init__(self, api_key=None): + super().__init__() + source_data = SourceService().get_source_data(code=self.code) + self.name = source_data["name"] + self.type = source_data["type"] + self.url = source_data["url"] + self.supported_metrics = source_data["supported_metrics"] self.api_key = api_key or os.environ.get("NEWSAPI_KEY") def supports_metric(self, metric: str) -> bool: return metric in self.supported_metrics - def fetch(self, app_id: int, metric: str): - try: - app = App.objects.get(id=app_id) - except App.DoesNotExist: - return None + def fetch(self, app_id: int, metrics: list[str]): + app = AppService().get_app(app_id=app_id) + if not app: + return {} if not self.api_key: - return None + return {} - today = datetime.today().strftime("%Y-%m-%d") + yesterday = (datetime.today() - timedelta(days=1)).strftime("%Y-%m-%d") params = { "q": app.code, - "from": today, - "to": today, + "from": yesterday, + "to": yesterday, "sortBy": "publishedAt", "language": "en", "apiKey": self.api_key, } response = requests.get(f"{self.url}/everything", params=params) + if not response.ok: - return None + return {} data = response.json() - articles = data.get("articles", []) + + metric_map = { + MetricCodes.DAILY_NEWS_BLOG_MENTIONS: str(data.get("totalResults", 0)), + } return { - MetricCodes.DAILY_NEWS_BLOG_MENTIONS: len(articles), - }.get(metric) + metric: metric_map.get(metric, "") for metric in metrics if self.supports_metric(metric) + } diff --git a/source/adapters/reddit.py b/source/adapters/reddit.py index 1de618f..9934d07 100644 --- a/source/adapters/reddit.py +++ b/source/adapters/reddit.py @@ -2,37 +2,37 @@ import praw -from app.models import App +from app.services import AppService from metric.constants import MetricCodes from source.adapters.base import SourceAdapter -from source.constants.source_type import SourceType +from source.services import SourceService class RedditAPIAdapter(SourceAdapter): code = "reddit" - name = "Reddit API" - type = SourceType.API - url = "https://oauth.reddit.com" - supported_metrics = [MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS] def __init__(self): + super().__init__() + source_data = SourceService().get_source_data(code=self.code) + self.name = source_data["name"] + self.type = source_data["type"] + self.url = source_data["url"] + self.supported_metrics = source_data["supported_metrics"] self.reddit = praw.Reddit( - client_id=os.environ.get("REDDIT_CLIENT_ID"), - client_secret=os.environ.get("REDDIT_CLIENT_SECRET"), - user_agent=os.environ.get("REDDIT_USER_AGENT"), + client_id=os.getenv("REDDIT_CLIENT_ID"), + client_secret=os.getenv("REDDIT_CLIENT_SECRET"), + user_agent=os.getenv("REDDIT_USER_AGENT"), + username=os.getenv("REDDIT_USERNAME"), + password=os.getenv("REDDIT_PASSWORD"), ) def supports_metric(self, metric: str) -> bool: return metric in self.supported_metrics - def fetch(self, app_id: int, metric: str): - if metric != MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS: - return None - - try: - app = App.objects.get(id=app_id) - except App.DoesNotExist: - return None + def fetch(self, app_id: int, metrics: list[str]): + app = AppService().get_app(app_id=app_id) + if not app: + return {} query = app.name.lower() total = 0 @@ -54,9 +54,13 @@ def fetch(self, app_id: int, metric: str): after = batch[-1].fullname # exemple: 't3_abcdef' except Exception as e: - print(f"[RedditAPIAdapter] Error fetching paginated data: {e}") - return None + print(f"[RedditAPIAdapter] Error fetching search data: {e}") + return {} + + metric_map = { + MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS: str(total), + } return { - MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS: total, - }.get(metric) + metric: metric_map.get(metric, "") for metric in metrics if self.supports_metric(metric) + } diff --git a/source/repositories.py b/source/repositories.py index b205643..ffe8610 100644 --- a/source/repositories.py +++ b/source/repositories.py @@ -4,30 +4,42 @@ class SourceRepository: - def get_all(self): + + @staticmethod + def get_all(): return Source.objects.all() - def get_by_id(self, app_id): + @staticmethod + def get_by_id(app_id): return get_object_or_404(Source, id=app_id) - def create(self, data): + @staticmethod + def create(data): return Source.objects.create(**data) - def update(self, instance, data): + @staticmethod + def update(instance, data): for attr, value in data.items(): setattr(instance, attr, value) instance.save() return instance - def delete(self, instance): + @staticmethod + def delete(instance): instance.delete() - def add_metrics(self, instance, metrics_ids): + @staticmethod + def add_metrics(instance, metrics_ids): instance.metrics.add(*metrics_ids) instance.save() return instance - def remove_metrics(self, instance, metrics_ids): + @staticmethod + def remove_metrics(instance, metrics_ids): instance.metrics.remove(*metrics_ids) instance.save() return instance + + @staticmethod + def get_by_code(code: str): + return Source.objects.prefetch_related("metrics").get(code=code) diff --git a/source/services.py b/source/services.py index cebba0c..d9fbd9c 100644 --- a/source/services.py +++ b/source/services.py @@ -2,14 +2,6 @@ from .repositories import SourceRepository -# isort: off -import source.adapters.itunes as _ # noqa -import source.adapters.news as _ # noqa -import source.adapters.reddit as _ # noqa -import source.adapters.google_play_scraper as _ # noqa - -# isort: on - class SourceService: def __init__(self): @@ -36,6 +28,23 @@ def add_metrics(self, instance, metrics_ids): def remove_metrics(self, instance, metrics_ids): return self.repo.remove_metrics(instance, metrics_ids) + def get_source_data(self, code: str) -> dict: + source = self.repo.get_by_code(code) + return { + "code": source.code, + "name": source.name, + "type": source.type, + "url": source.url, + "supported_metrics": list(source.metrics.values_list("code", flat=True)), + } + @staticmethod def load_sources(): + # isort: off + import source.adapters.itunes as _ # noqa + import source.adapters.news as _ # noqa + import source.adapters.reddit as _ # noqa + import source.adapters.google_play_scraper as _ # noqa + + # isort: on return [cls() for cls in SourceAdapter.__subclasses__()] diff --git a/tests/adapters/__init__.py b/tests/adapters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/adapters/test_all_adapters.py b/tests/adapters/test_all_adapters.py new file mode 100644 index 0000000..7015a2f --- /dev/null +++ b/tests/adapters/test_all_adapters.py @@ -0,0 +1,27 @@ +import pytest + +from source.adapters.base import SourceAdapter + +# isort: off +import source.adapters.itunes as _ # noqa +import source.adapters.news as _ # noqa +import source.adapters.reddit as _ # noqa +import source.adapters.google_play_scraper as _ # noqa + + +# isort: on + + +@pytest.mark.django_db +@pytest.mark.parametrize("adapter_class", SourceAdapter.__subclasses__()) +def test_fetch_returns_dict_of_strings(adapter_class, dummy_app): + + adapter = adapter_class() + if not adapter.supported_metrics: + pytest.skip(f"No supported metrics for {adapter.code}") + result = adapter.fetch(app_id=dummy_app.id, metrics=adapter.supported_metrics) + assert isinstance(result, dict) + assert result is not None, f"{adapter.code}.fetch() returned None" + for k, v in result.items(): + assert isinstance(k, str), f"Key {k} in {adapter.code} is not a string" + assert isinstance(v, str), f"Value for {k} in {adapter.code} is not a string" diff --git a/tests/adapters/test_google_play_scraper.py b/tests/adapters/test_google_play_scraper.py new file mode 100644 index 0000000..45027d0 --- /dev/null +++ b/tests/adapters/test_google_play_scraper.py @@ -0,0 +1,127 @@ +from datetime import datetime +from unittest.mock import patch + +import pytest +from rest_framework.exceptions import ParseError, ValidationError + +from metric.constants import MetricCodes +from source.adapters.google_play_scraper import GooglePlayScraperAdapter + + +@patch("app.models.App.objects.get") +@patch("source.adapters.google_play_scraper.gp_app") +def test_google_play_fetch(mock_gp_app, mock_app_get, dummy_app): + """Testa que GooglePlayScraperAdapter.fetch retorna valors esperats amb mocks.""" + + mock_app_get.return_value.playstore_id = "com.whatsapp" + mock_gp_app.return_value = { + "score": 4.6, + "reviews": 50000, + "realInstalls": 1000000, + "lastUpdatedOn": "2024-12-10", + } + + adapter = GooglePlayScraperAdapter() + + assert adapter.supports_metric(MetricCodes.AVERAGE_RATING) + + result = adapter.fetch( + dummy_app.id, + [ + MetricCodes.AVERAGE_RATING, + MetricCodes.TOTAL_REVIEWS, + MetricCodes.TOTAL_DOWNLOADS, + MetricCodes.LAST_UPDATE_DATE, + ], + ) + + assert result == { + MetricCodes.AVERAGE_RATING: "4.6", + MetricCodes.TOTAL_REVIEWS: "50000", + MetricCodes.TOTAL_DOWNLOADS: "1000000", + MetricCodes.LAST_UPDATE_DATE: "2024-12-10", + } + + unsupported = adapter.fetch(dummy_app.id, ["unsupported_metric"]) + assert unsupported == {} + + +@patch("app.models.App.objects.get") +@patch("source.adapters.google_play_scraper.reviews") +def test_google_play_fetch_reviews(mock_reviews, mock_app_get, dummy_app): + """Testa que GooglePlayScraperAdapter.fetch_reviews retorna reviews dins del rang.""" + + mock_app_get.return_value.playstore_id = "com.whatsapp" + + # Simulem que es retornen dues tandes de reviews, amb una data dins el rang + mock_reviews.side_effect = [ + ( + [ + {"userName": "Anna", "content": "Molt bona!", "at": datetime(2024, 5, 3)}, + {"userName": "Pau", "content": "No funciona bé", "at": datetime(2024, 5, 2)}, + ], + "token123", + ), + ( + [ + {"userName": "Joan", "content": "Ok", "at": datetime(2024, 5, 1)}, + ], + None, + ), + ] + + adapter = GooglePlayScraperAdapter() + result = adapter.fetch_reviews(dummy_app.id, date_from="2024-05-01", date_to="2024-05-03") + + # Ha de contenir les 3 reviews simulades + assert isinstance(result, list) + assert len(result) == 3 + assert all("content" in r for r in result) + assert all("userName" in r for r in result) + + +@patch("app.models.App.objects.get") +@patch("source.adapters.google_play_scraper.reviews") +def test_fetch_reviews_stops_on_review_before_date_from(mock_reviews, mock_app_get, dummy_app): + """Ha de tallar si una review és anterior a date_from.""" + + mock_app_get.return_value.playstore_id = "com.whatsapp" + mock_reviews.return_value = ( + [{"content": "Antiga", "at": datetime(2024, 4, 30)}], # fora del rang + None, + ) + + adapter = GooglePlayScraperAdapter() + result = adapter.fetch_reviews(dummy_app.id, date_from="2024-05-01", date_to="2024-05-03") + + assert result == [] # Ha de sortir immediatament + + +@patch("app.models.App.objects.get") +@patch("source.adapters.google_play_scraper.reviews") +def test_fetch_reviews_returns_empty_list_if_no_reviews(mock_reviews, mock_app_get, dummy_app): + mock_app_get.return_value.playstore_id = "com.whatsapp" + mock_reviews.return_value = ([], None) + + adapter = GooglePlayScraperAdapter() + result = adapter.fetch_reviews(dummy_app.id) + + assert result == [] + + +@patch("app.models.App.objects.get") +def test_fetch_reviews_raises_parse_error_on_invalid_date(mock_app_get, dummy_app): + mock_app_get.return_value.playstore_id = "com.whatsapp" + adapter = GooglePlayScraperAdapter() + + with pytest.raises(ParseError): + adapter.fetch_reviews(dummy_app.id, date_from="invalid-date") + + +@patch("app.models.App.objects.get") +def test_fetch_reviews_raises_validation_error_if_date_range_invalid(mock_app_get, dummy_app): + mock_app_get.return_value.playstore_id = "com.whatsapp" + adapter = GooglePlayScraperAdapter() + + with pytest.raises(ValidationError): + adapter.fetch_reviews(dummy_app.id, date_from="2024-05-05", date_to="2024-05-01") diff --git a/tests/adapters/test_itunes_adapter.py b/tests/adapters/test_itunes_adapter.py new file mode 100644 index 0000000..ba448ef --- /dev/null +++ b/tests/adapters/test_itunes_adapter.py @@ -0,0 +1,26 @@ +from unittest.mock import patch + +from metric.constants import MetricCodes +from source.adapters.itunes import ItunesSearchAPIAdapter + + +@patch("app.models.App.objects.get") +@patch("source.adapters.itunes.requests.get") +def test_itunes_fetch(mock_requests_get, mock_app_get, dummy_app): + """Testa que ItunesSearchAPIAdapter.fetch retorna valors esperats amb mocks.""" + mock_app_get.return_value.appstore_id = "123" + mock_requests_get.return_value.ok = True + mock_requests_get.return_value.json.return_value = { + "resultCount": 1, + "results": [{"averageUserRating": 4.5, "userRatingCount": 1000}], + } + + adapter = ItunesSearchAPIAdapter() + + assert adapter.supports_metric(MetricCodes.AVERAGE_RATING) + + result = adapter.fetch(dummy_app.id, [MetricCodes.AVERAGE_RATING, MetricCodes.TOTAL_REVIEWS]) + assert result == {MetricCodes.AVERAGE_RATING: "4.5", MetricCodes.TOTAL_REVIEWS: "1000"} + + unsupported = adapter.fetch(dummy_app.id, ["unsupported_metric"]) + assert unsupported == {} diff --git a/tests/adapters/test_news.py b/tests/adapters/test_news.py new file mode 100644 index 0000000..39cb929 --- /dev/null +++ b/tests/adapters/test_news.py @@ -0,0 +1,31 @@ +from unittest.mock import patch + +from metric.constants import MetricCodes +from source.adapters.news import NewsAPIAdapter + + +@patch("app.models.App.objects.get") +@patch("source.adapters.news.requests.get") +def test_news_adapter_fetch(mock_requests_get, mock_app_get, dummy_app): + """Testa que NewsAPIAdapter.fetch retorna valors esperats amb mocks.""" + + # Simulem app amb codi "whatsapp" + mock_app_get.return_value.code = "whatsapp" + + # Simulem resposta de l'API de NewsAPI + mock_requests_get.return_value.ok = True + mock_requests_get.return_value.json.return_value = { + "status": "ok", + "totalResults": 42, + "articles": [{}] * 42, + } + + adapter = NewsAPIAdapter("fake-api-key") + + result = adapter.fetch(dummy_app.id, [MetricCodes.DAILY_NEWS_BLOG_MENTIONS]) + + assert isinstance(result, dict) + assert result.get(MetricCodes.DAILY_NEWS_BLOG_MENTIONS) == "42" + + unsupported = adapter.fetch(dummy_app.id, ["unsupported_metric"]) + assert unsupported == {} diff --git a/tests/adapters/test_reddit.py b/tests/adapters/test_reddit.py new file mode 100644 index 0000000..625b778 --- /dev/null +++ b/tests/adapters/test_reddit.py @@ -0,0 +1,31 @@ +from unittest.mock import MagicMock, patch + +from metric.constants import MetricCodes +from source.adapters.reddit import RedditAPIAdapter + + +@patch("app.models.App.objects.get") +@patch("praw.Reddit") +def test_reddit_adapter_fetch(mock_reddit_class, mock_app_get, dummy_app): + """Testa que RedditAPIAdapter.fetch retorna valors esperats amb mocks.""" + + # Simulem app + mock_app_get.return_value.name = "WhatsApp" + + # Simulem la instància Reddit i el seu comportament + mock_reddit = MagicMock() + mock_results = [{}] * 23 # 23 mencions + mock_reddit.subreddit.return_value.search.return_value = mock_results + mock_reddit_class.return_value = mock_reddit + + metric_code = MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS + + # ✅ Instanciar adapter després de tenir la BD a punt + adapter = RedditAPIAdapter() + + result = adapter.fetch(dummy_app.id, [metric_code]) + assert isinstance(result, dict) + assert result.get(metric_code) == "23" + + unsupported = adapter.fetch(dummy_app.id, ["unsupported_metric"]) + assert unsupported == {} diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9b107a7 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,43 @@ +import pytest + +from app.models import App +from metric.constants import MetricCodes +from metric.models import Metric +from source.models import Source + + +@pytest.fixture +def dummy_app(db): + return App.objects.create( + code="dummy", name="Dummy App", appstore_id="310633997", playstore_id="com.whatsapp" + ) + + +@pytest.fixture(autouse=True) +def create_default_sources_and_metrics(db): + """Crea sources i mètriques per defecte disponibles a tots els tests.""" + + itunes = Source.objects.create( + code="itunes", name="App Store", type="api", url="https://itunes.apple.com" + ) + google = Source.objects.create( + code="google_play", name="Google Play", type="scraper", url="https://play.google.com" + ) + news = Source.objects.create(code="news", name="NewsAPI", type="api", url="https://newsapi.org") + reddit = Source.objects.create( + code="reddit", name="Reddit", type="api", url="https://reddit.com" + ) + + for metric_code in [ + MetricCodes.AVERAGE_RATING, + MetricCodes.TOTAL_REVIEWS, + MetricCodes.TOTAL_DOWNLOADS, + MetricCodes.LAST_UPDATE_DATE, + MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS, + MetricCodes.DAILY_NEWS_BLOG_MENTIONS, + ]: + metric = Metric.objects.create(code=metric_code, name=metric_code.replace("_", " ").title()) + itunes.metrics.add(metric) + google.metrics.add(metric) + news.metrics.add(metric) + reddit.metrics.add(metric) diff --git a/tests/unit/test_google_play_scraper_adapter.py b/tests/unit/test_google_play_scraper_adapter.py deleted file mode 100644 index 8c29529..0000000 --- a/tests/unit/test_google_play_scraper_adapter.py +++ /dev/null @@ -1,27 +0,0 @@ -from unittest.mock import patch - -import pytest - -from app.models import App -from metric.constants import MetricCodes -from source.adapters.google_play_scraper import GooglePlayScraperAdapter - - -@pytest.mark.django_db -@patch("source.adapters.google_play_scraper.gp_app") -def test_google_play_scraper_adapter_fetch(mock_gp_app): - app = App.objects.create(name="Discord", playstore_id="com.discord") - - # Simulem la resposta de google_play_scraper.app() - mock_gp_app.return_value = {"score": 4.6, "reviews": 7890} - - adapter = GooglePlayScraperAdapter() - - assert adapter.supports_metric(MetricCodes.AVERAGE_RATING) - assert adapter.supports_metric(MetricCodes.TOTAL_REVIEWS) - - rating = adapter.fetch(app.id, MetricCodes.AVERAGE_RATING) - reviews = adapter.fetch(app.id, MetricCodes.TOTAL_REVIEWS) - - assert rating == 4.6 - assert reviews == 7890 diff --git a/tests/unit/test_itunes_adapter.py b/tests/unit/test_itunes_adapter.py deleted file mode 100644 index e214e47..0000000 --- a/tests/unit/test_itunes_adapter.py +++ /dev/null @@ -1,21 +0,0 @@ -from unittest.mock import patch - -from metric.constants import MetricCodes -from source.adapters.itunes import ItunesSearchAPIAdapter - - -@patch("source.adapters.itunes.App.objects.get") -@patch("source.adapters.itunes.requests.get") -def test_itunes_fetch(mock_get, mock_app_get): - mock_app_get.return_value.appstore_id = "123" - mock_get.return_value.ok = True - mock_get.return_value.json.return_value = { - "resultCount": 1, - "results": [{"averageUserRating": 4.5, "userRatingCount": 1000}], - } - - adapter = ItunesSearchAPIAdapter() - - assert adapter.supports_metric(MetricCodes.AVERAGE_RATING) - assert adapter.fetch("my_app", MetricCodes.AVERAGE_RATING) == 4.5 - assert adapter.fetch("my_app", MetricCodes.TOTAL_REVIEWS) == 1000 diff --git a/tests/unit/test_news_adapter.py b/tests/unit/test_news_adapter.py deleted file mode 100644 index 0667f9e..0000000 --- a/tests/unit/test_news_adapter.py +++ /dev/null @@ -1,30 +0,0 @@ -from unittest.mock import MagicMock, patch - -import pytest - -from app.models import App -from metric.constants import MetricCodes -from source.adapters.news import NewsAPIAdapter - - -@pytest.mark.django_db -@patch("source.adapters.news.requests.get") -def test_news_api_adapter_fetch(mock_get): - app = App.objects.create( - code="discord", - name="Discord", - appstore_id="985746746", - playstore_id="com.discord", - description="App de comunicació global", - ) - - mock_response = MagicMock() - mock_response.ok = True - mock_response.json.return_value = {"articles": [{}, {}, {}]} - mock_get.return_value = mock_response - - adapter = NewsAPIAdapter(api_key="fake_key") - - result = adapter.fetch(app.id, MetricCodes.DAILY_NEWS_BLOG_MENTIONS) - - assert result == 3 diff --git a/tests/unit/test_reddit_adapter.py b/tests/unit/test_reddit_adapter.py deleted file mode 100644 index 60874ad..0000000 --- a/tests/unit/test_reddit_adapter.py +++ /dev/null @@ -1,34 +0,0 @@ -from unittest.mock import MagicMock, patch - -import pytest - -from app.models import App -from metric.constants import MetricCodes -from source.adapters.reddit import RedditAPIAdapter - - -@pytest.mark.django_db -@patch("source.adapters.reddit.praw.Reddit") -def test_reddit_api_adapter_fetch_mentions(mock_praw): - app = App.objects.create( - code="discord", - name="Discord", - appstore_id="985746746", - playstore_id="com.discord", - description="App de comunicació global", - ) - - # Mock del resultat de la crida a Reddit - mock_instance = MagicMock() - mock_results = [1, 2, 3] # Simulem 3 resultats trobats - mock_instance.subreddit.return_value.search.return_value = mock_results - mock_praw.return_value = mock_instance - - adapter = RedditAPIAdapter() - - # Comprovació de supports_metric - assert adapter.supports_metric(MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS) is True - - # Comprovació del fetch (amb resultats simulats) - count = adapter.fetch(app.id, MetricCodes.DAILY_SOCIAL_NETWORK_MENTIONS) - assert count == 3 From 68aaf04c370d4431eab6d3e14e1a31b4ecebb66d Mon Sep 17 00:00:00 2001 From: Anyer Date: Sun, 4 May 2025 21:54:06 +0200 Subject: [PATCH 33/42] =?UTF-8?q?feature(metrics):=20Implementar=20funci?= =?UTF-8?q?=C3=B3=20per=20a=20la=20recollida=20peri=C3=B2dica=20de=20m?= =?UTF-8?q?=C3=A8triques?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #115 --- config/settings.py | 1 + config/urls.py | 1 + polling/__init__.py | 0 polling/admin.py | 5 +++ polling/apps.py | 6 ++++ polling/migrations/0001_initial.py | 54 ++++++++++++++++++++++++++++++ polling/migrations/__init__.py | 0 polling/models.py | 22 ++++++++++++ polling/repositories.py | 17 ++++++++++ polling/serializers.py | 10 ++++++ polling/services.py | 12 +++++++ polling/tests.py | 0 polling/urls.py | 13 +++++++ polling/views.py | 45 +++++++++++++++++++++++++ review/views.py | 2 +- 15 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 polling/__init__.py create mode 100644 polling/admin.py create mode 100644 polling/apps.py create mode 100644 polling/migrations/0001_initial.py create mode 100644 polling/migrations/__init__.py create mode 100644 polling/models.py create mode 100644 polling/repositories.py create mode 100644 polling/serializers.py create mode 100644 polling/services.py create mode 100644 polling/tests.py create mode 100644 polling/urls.py create mode 100644 polling/views.py diff --git a/config/settings.py b/config/settings.py index 16eab80..f6dccec 100644 --- a/config/settings.py +++ b/config/settings.py @@ -49,6 +49,7 @@ "source", "review", "metric", + "polling", "drf_spectacular", ] diff --git a/config/urls.py b/config/urls.py index 60775e6..a47e27e 100644 --- a/config/urls.py +++ b/config/urls.py @@ -34,4 +34,5 @@ def home(request): path("api/", include("source.urls")), path("api/", include("review.urls")), path("api/", include("metric.urls")), + path("api/polling/", include("polling.urls")), ] diff --git a/polling/__init__.py b/polling/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/polling/admin.py b/polling/admin.py new file mode 100644 index 0000000..3fed5f0 --- /dev/null +++ b/polling/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin + +from polling.models import PollingSchedule + +admin.site.register(PollingSchedule) diff --git a/polling/apps.py b/polling/apps.py new file mode 100644 index 0000000..75ce417 --- /dev/null +++ b/polling/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class PollingConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "polling" diff --git a/polling/migrations/0001_initial.py b/polling/migrations/0001_initial.py new file mode 100644 index 0000000..9c5b805 --- /dev/null +++ b/polling/migrations/0001_initial.py @@ -0,0 +1,54 @@ +# Generated by Django 5.1.7 on 2025-05-04 19:10 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("app", "0003_alter_app_developer"), + ] + + operations = [ + migrations.CreateModel( + name="PollingSchedule", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("interval_hours", models.IntegerField()), + ("start_at", models.DateTimeField(auto_now_add=True)), + ("next_run", models.DateTimeField()), + ( + "poll_type", + models.CharField( + choices=[ + ("metrics", "Metrics only"), + ("reviews", "Reviews only"), + ("both", "Metrics + Reviews"), + ], + default="metrics", + max_length=10, + ), + ), + ("is_active", models.BooleanField(default=True)), + ( + "app", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="polling_schedules", + to="app.app", + ), + ), + ], + options={ + "unique_together": {("app", "poll_type")}, + }, + ), + ] diff --git a/polling/migrations/__init__.py b/polling/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/polling/models.py b/polling/models.py new file mode 100644 index 0000000..787cff3 --- /dev/null +++ b/polling/models.py @@ -0,0 +1,22 @@ +from django.db import models + +from app.models import App + + +class PollingSchedule(models.Model): + app = models.ForeignKey(App, on_delete=models.CASCADE, related_name="polling_schedules") + interval_hours = models.IntegerField() + start_at = models.DateTimeField(auto_now_add=True) + next_run = models.DateTimeField() + + POLL_TYPE_CHOICES = [ + ("metrics", "Metrics only"), + ("reviews", "Reviews only"), + ("both", "Metrics + Reviews"), + ] + poll_type = models.CharField(max_length=10, choices=POLL_TYPE_CHOICES, default="metrics") + + is_active = models.BooleanField(default=True) + + class Meta: + unique_together = ("app", "poll_type") diff --git a/polling/repositories.py b/polling/repositories.py new file mode 100644 index 0000000..248cbd3 --- /dev/null +++ b/polling/repositories.py @@ -0,0 +1,17 @@ +from .models import PollingSchedule + + +class PollingRepository: + @staticmethod + def get_all_active(poll_type): + queryset = PollingSchedule.objects.filter(is_active=True) + if poll_type: + queryset = queryset.filter(poll_type=poll_type) + return queryset + + @staticmethod + def get_by_app_id(app_id, poll_type): + queryset = PollingSchedule.objects.filter(app_id=app_id, is_active=True) + if poll_type: + queryset = queryset.filter(poll_type=poll_type) + return queryset.first() diff --git a/polling/serializers.py b/polling/serializers.py new file mode 100644 index 0000000..e8a9333 --- /dev/null +++ b/polling/serializers.py @@ -0,0 +1,10 @@ +from rest_framework import serializers + +from polling.models import PollingSchedule + + +class PollingScheduleSerializer(serializers.ModelSerializer): + class Meta: + model = PollingSchedule + fields = ["app", "interval_hours", "start_at", "next_run", "poll_type", "is_active"] + read_only_fields = ["start_at", "next_run", "is_active"] diff --git a/polling/services.py b/polling/services.py new file mode 100644 index 0000000..054ce70 --- /dev/null +++ b/polling/services.py @@ -0,0 +1,12 @@ +from .repositories import PollingRepository + + +class PollingService: + def __init__(self): + self.repo = PollingRepository() + + def list_active_polling_schedules(self, poll_type=None): + return self.repo.get_all_active(poll_type) + + def get_polling_schedule(self, app_id, poll_type=None): + return self.repo.get_by_app_id(app_id, poll_type) diff --git a/polling/tests.py b/polling/tests.py new file mode 100644 index 0000000..e69de29 diff --git a/polling/urls.py b/polling/urls.py new file mode 100644 index 0000000..dd17aa6 --- /dev/null +++ b/polling/urls.py @@ -0,0 +1,13 @@ +from django.urls import path + +from polling.views import PollingScheduleViewSet + +polling_retrieve_view = PollingScheduleViewSet.as_view({"get": "retrieve"}) +polling_list_view = PollingScheduleViewSet.as_view({"get": "list"}) + +urlpatterns = [ + path("metrics/", polling_list_view, {"poll_type": "metrics"}), + path("reviews/", polling_list_view, {"poll_type": "reviews"}), + path("metrics/apps//", polling_retrieve_view, {"poll_type": "metrics"}), + path("reviews/apps//", polling_retrieve_view, {"poll_type": "reviews"}), +] diff --git a/polling/views.py b/polling/views.py new file mode 100644 index 0000000..333ab8e --- /dev/null +++ b/polling/views.py @@ -0,0 +1,45 @@ +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import OpenApiParameter, extend_schema +from rest_framework import status, viewsets +from rest_framework.response import Response + +from polling.serializers import PollingScheduleSerializer +from polling.services import PollingService + + +class PollingScheduleViewSet(viewsets.ViewSet): + service = PollingService() + + @extend_schema( + description="List all active metric polling schedules.", + responses={200: PollingScheduleSerializer(many=True)}, + tags=["Polling"], + ) + def list(self, request, poll_type=None): + schedules = self.service.list_active_polling_schedules(poll_type) + serializer = PollingScheduleSerializer(schedules, many=True) + return Response(serializer.data, status=status.HTTP_200_OK) + + @extend_schema( + description="Check if there is an active polling schedule for an app.", + parameters=[ + OpenApiParameter( + name="id", required=True, type=OpenApiTypes.INT, location=OpenApiParameter.PATH + ), + ], + responses={200: PollingScheduleSerializer}, + tags=["Polling"], + ) + def retrieve(self, request, id=None, poll_type=None): + schedule = self.service.get_polling_schedule(id, poll_type) + if not schedule: + if poll_type == "metrics": + message = f"There is no active polling schedule for metrics for app {id}." + elif poll_type == "reviews": + message = f"There is no active polling schedule for reviews for app {id}." + else: + message = f"No active polling schedule found for app {id}." + return Response({"detail": message}, status=status.HTTP_404_NOT_FOUND) + + serializer = PollingScheduleSerializer(schedule) + return Response(serializer.data, status=status.HTTP_200_OK) diff --git a/review/views.py b/review/views.py index 2e354ab..80a1d7c 100644 --- a/review/views.py +++ b/review/views.py @@ -34,7 +34,7 @@ def list(self, request): tags=["Reviews"], ) def retrieve(self, request, pk=None): - source = self.service.get_source(pk) + source = self.service.get_review(pk) serializer = ReviewSerializer(source) return Response(serializer.data) From d1094cd0dadc2cc673132a8fd2a6ab574b237005 Mon Sep 17 00:00:00 2001 From: Anyerrr Date: Mon, 5 May 2025 10:20:40 +0200 Subject: [PATCH 34/42] =?UTF-8?q?feat(celery):=20Implementar=20funci=C3=B3?= =?UTF-8?q?=20per=20a=20la=20recollida=20peri=C3=B2dica=20de=20m=C3=A8triq?= =?UTF-8?q?ues?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #115 --- config/__init__.py | 3 + config/celery.py | 8 + config/settings.py | 4 + docker-compose.yml | 33 +- polling/repositories.py | 36 +- polling/serializers.py | 4 +- polling/services.py | 41 +- polling/tasks.py | 10 + polling/urls.py | 8 +- polling/views.py | 67 ++- requirements.txt | Bin 1756 -> 1139 bytes review/services.py | 11 - review/urls.py | 5 - review/views.py | 42 -- schema.yaml | 1237 +++++++++++++++++++++++++++++++++++++++ 15 files changed, 1441 insertions(+), 68 deletions(-) create mode 100644 config/celery.py create mode 100644 polling/tasks.py create mode 100644 schema.yaml diff --git a/config/__init__.py b/config/__init__.py index e69de29..e31568a 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ("celery_app",) \ No newline at end of file diff --git a/config/celery.py b/config/celery.py new file mode 100644 index 0000000..d33bb1a --- /dev/null +++ b/config/celery.py @@ -0,0 +1,8 @@ +import os +from celery import Celery + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') + +app = Celery('metaappcollector') +app.config_from_object('django.conf:settings', namespace='CELERY') +app.autodiscover_tasks() diff --git a/config/settings.py b/config/settings.py index f6dccec..dfe1c00 100644 --- a/config/settings.py +++ b/config/settings.py @@ -51,6 +51,7 @@ "metric", "polling", "drf_spectacular", + "django_celery_beat", ] MIDDLEWARE = [ @@ -139,3 +140,6 @@ REST_FRAMEWORK = { "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", } + +CELERY_BROKER_URL = 'redis://localhost:6379/0' +CELERY_RESULT_BACKEND = 'redis://localhost:6379/0' diff --git a/docker-compose.yml b/docker-compose.yml index 903bb4e..fa3bbbd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,39 @@ +version: '3.8' services: + redis: + image: redis:7 + container_name: redis + ports: + - "6379:6379" + web: build: . + container_name: django + command: python manage.py runserver 0.0.0.0:8000 + volumes: + - .:/app ports: - "8000:8000" + depends_on: + - redis + env_file: + - .env + + celery: + build: . + container_name: celery_worker + command: celery -A config worker --loglevel=info volumes: - .:/app - env_file: - - .env.prod + depends_on: + - redis + + beat: + build: . + container_name: celery_beat + command: celery -A config beat --loglevel=info --scheduler django_celery_beat.schedulers:DatabaseScheduler + volumes: + - .:/app + depends_on: + - redis diff --git a/polling/repositories.py b/polling/repositories.py index 248cbd3..b24d46c 100644 --- a/polling/repositories.py +++ b/polling/repositories.py @@ -1,5 +1,6 @@ from .models import PollingSchedule - +from django_celery_beat.models import PeriodicTask, IntervalSchedule +import json class PollingRepository: @staticmethod @@ -15,3 +16,36 @@ def get_by_app_id(app_id, poll_type): if poll_type: queryset = queryset.filter(poll_type=poll_type) return queryset.first() + + @staticmethod + def create(app_id, interval_hours, poll_type): + from django.utils.timezone import now + from datetime import timedelta + + start = now() + schedule = PollingSchedule.objects.create( + app_id=app_id, + interval_hours=interval_hours, + start_at=start, + next_run=start + timedelta(hours=interval_hours), + poll_type=poll_type, + is_active=True, + ) + return schedule + + @staticmethod + def get_or_create_interval_schedule(interval_hours): + schedule, _ = IntervalSchedule.objects.get_or_create( + every=interval_hours, + period=IntervalSchedule.HOURS, + ) + return schedule + + @staticmethod + def create_periodic_task(schedule, app_id, poll_type, task): + PeriodicTask.objects.create( + interval=schedule, + name=f"{poll_type}-polling-{app_id}", + task=task, + args=json.dumps([app_id, poll_type]), + ) \ No newline at end of file diff --git a/polling/serializers.py b/polling/serializers.py index e8a9333..fd5510e 100644 --- a/polling/serializers.py +++ b/polling/serializers.py @@ -6,5 +6,5 @@ class PollingScheduleSerializer(serializers.ModelSerializer): class Meta: model = PollingSchedule - fields = ["app", "interval_hours", "start_at", "next_run", "poll_type", "is_active"] - read_only_fields = ["start_at", "next_run", "is_active"] + fields = ["app", "interval_hours", "start_at", "next_run", "is_active"] + read_only_fields = ["app", "start_at", "next_run", "is_active"] diff --git a/polling/services.py b/polling/services.py index 054ce70..b2fcb02 100644 --- a/polling/services.py +++ b/polling/services.py @@ -1,12 +1,51 @@ from .repositories import PollingRepository - +from source.services import SourceService +from review.repositories import ReviewRepository +from polling.tasks import run_polling_task +from celery import current_app +from datetime import timedelta +from django.utils.timezone import now class PollingService: def __init__(self): self.repo = PollingRepository() + self.source_service = SourceService() def list_active_polling_schedules(self, poll_type=None): return self.repo.get_all_active(poll_type) def get_polling_schedule(self, app_id, poll_type=None): return self.repo.get_by_app_id(app_id, poll_type) + + def poll_reviews(self, app_id, date_from=None, date_to=None): + adapters = self.source_service.load_sources() + results = [] + + for adapter in adapters: + reviews = adapter.fetch_reviews(app_id=app_id, date_from=date_from, date_to=date_to) + saved_count = ReviewRepository().save_reviews(app_id, reviews) + + results.append({ + "adapter": adapter.name, + "fetched": len(reviews), + "saved": saved_count, + "reviews": reviews, + }) + + return results + + def create_polling_schedule(self, app_id, interval_hours=None, poll_type=None): + existing = self.get_polling_schedule(app_id, poll_type) + if existing: + raise ValueError(f"Polling already scheduled for app {app_id} and type {poll_type}") + + schedule = self.repo.create_polling_schedule(app_id,interval_hours,poll_type) + + schedule2 = self.repo.get_or_create_interval_schedule(interval_hours) + + self.repo.create_perdioc_task(schedule2,app_id,poll_type,"polling.tasks.run_polling_task") + + return schedule + + def poll_metrics(self, app_id): + ... diff --git a/polling/tasks.py b/polling/tasks.py new file mode 100644 index 0000000..235f551 --- /dev/null +++ b/polling/tasks.py @@ -0,0 +1,10 @@ +from celery import shared_task + +@shared_task +def run_polling_task(app_id, poll_type): + from polling.services import PollingService + service = PollingService() + if poll_type == "metrics": + result = service.poll_metrics(app_id) + elif poll_type == "reviews": + result = service.poll_reviews(app_id) \ No newline at end of file diff --git a/polling/urls.py b/polling/urls.py index dd17aa6..335ba1a 100644 --- a/polling/urls.py +++ b/polling/urls.py @@ -4,10 +4,14 @@ polling_retrieve_view = PollingScheduleViewSet.as_view({"get": "retrieve"}) polling_list_view = PollingScheduleViewSet.as_view({"get": "list"}) +polling_detail_view = PollingScheduleViewSet.as_view({ + "get": "retrieve", + "post": "create" +}) urlpatterns = [ path("metrics/", polling_list_view, {"poll_type": "metrics"}), path("reviews/", polling_list_view, {"poll_type": "reviews"}), - path("metrics/apps//", polling_retrieve_view, {"poll_type": "metrics"}), - path("reviews/apps//", polling_retrieve_view, {"poll_type": "reviews"}), + path("metrics/apps//", polling_detail_view, {"poll_type": "metrics"}), + path("reviews/apps//", polling_detail_view, {"poll_type": "reviews"}), ] diff --git a/polling/views.py b/polling/views.py index 333ab8e..4ce3cde 100644 --- a/polling/views.py +++ b/polling/views.py @@ -1,11 +1,11 @@ from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import OpenApiParameter, extend_schema +from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_schema from rest_framework import status, viewsets from rest_framework.response import Response from polling.serializers import PollingScheduleSerializer from polling.services import PollingService - +from app.services import AppService class PollingScheduleViewSet(viewsets.ViewSet): service = PollingService() @@ -43,3 +43,66 @@ def retrieve(self, request, id=None, poll_type=None): serializer = PollingScheduleSerializer(schedule) return Response(serializer.data, status=status.HTTP_200_OK) + + @extend_schema( + request=PollingScheduleSerializer, + responses={201: PollingScheduleSerializer}, + tags=["Polling"], + ) + def create(self, request, id=None, poll_type=None): + serializer = PollingScheduleSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + interval = serializer.validated_data["interval_hours"] + + app = AppService().get_app(id) + if not app: + return Response({"detail": "App not found"}, status=404) + + try: + schedule = self.service.create_polling_schedule(app, interval, poll_type) + except ValueError as e: + return Response({"detail": str(e)}, status=409) + + return Response(PollingScheduleSerializer(schedule).data, status=201) + + @extend_schema( + request=None, + parameters=[ + OpenApiParameter( + name="id", + type=int, + required=True, + location="path", + description="App ID to fetch reviews for", + ), + OpenApiParameter( + name="date_from", type=str, description="Start date (YYYY-MM-DD)", required=False + ), + OpenApiParameter( + name="date_to", type=str, description="End date (YYYY-MM-DD)", required=False + ), + ], + responses={ + 200: OpenApiResponse(description="Reviews successfully fetched and saved."), + 400: OpenApiResponse( + description="Invalid request. Possible causes: missing parameters, " + "wrong date format, or invalid date range." + ), + 404: OpenApiResponse(description="App with the given ID was not found."), + }, + tags=["Polling"], + description=( + "Polls Google Play for reviews of a given app between two dates.\n\n" + "Saves new ones in the database and returns" + "the fetched reviews with stats.\n\n" + "If no `date_from` and `date_to` are provided," + " reviews from **yesterday** will be fetched." + ), + ) + def poll_reviews(self, request, id=None): + date_from = request.query_params.get("date_from") + date_to = request.query_params.get("date_to") + + result = self.service.poll_reviews(app_id=id, date_from=date_from, date_to=date_to) + + return Response(result) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 7c937320637ceac7443cdc35ec5d98210c9de846..a982ccad27118f951559d803247773915b68e896 100644 GIT binary patch literal 1139 zcmYjQxo+Gr5bgOdB6RFw7b*gzOn|hh42l|Q$4ikkNqH@Qecnh85citf0EDZzzg79;1Az9^f5uSUu?aV!BN8-&kl zs&XV-j9s0TK_v!@iC%1276vqNt3<*Q%S#Et-6aZa`i#P@KqfI< z2P+(>>d;Rr8*vt*H#kS^Qw)649Sx7@XbP=`VY(K;Y8^u^nD&+Y5?dTvt6$siZ$Ez= zT(=7Ej1qafa{ymoz(S(873&DMy2GbZ9CH-`8NLu)nLl>FU*-SJMR`V�ynri|F<8 zKlK2BWmWmamXLH8tW|52r7P%c?_wh`E*4u87g~gOv_vJ4uL)0n==ODRYxn41|3~;->^o-R=|1!*bH zR9o2DD~DPjR(7k;&~mQWQ90e91Xb_-4ez-ymQvtgy>c35qWeiK(I>m8eR6?^<{RbG zZE6uUQ6zHcVW&>W8wnY-i&tr!GqnPp4r{%m%T|0~8neB4eWG*|Dv{9%EvHir62OmPedc%;7a?^}6A z3pa;I<0eaHxV zh2L{dpqFPC_UO~xY7(vZlv$Lli>t=I5Tyd_ZhWgy7EOtcq+8jsM5>%|YDe~3j5^Q$ zf3N5)W`RQAh5W-l+c*2_b<;<+*Nv;l4lt+YY7%i1$S#@BQZP8CPv`;HP6o7I+z-BK zp)x4-)xA5|v*y5L?aHy_s%Srs0BbyahgTKox(#7O$^x~3EJRWM~)L#@~gHPhd4a(aGKUCv0 ZTy$1+Chg*u#jio!d7cW-Fx7I~{s9aw0-gW> diff --git a/review/services.py b/review/services.py index 22a3a8b..be1520f 100644 --- a/review/services.py +++ b/review/services.py @@ -6,7 +6,6 @@ class ReviewService: def __init__(self): self.repo = ReviewRepository() - self.adapter = GooglePlayScraperAdapter() def list_reviews(self, filters=None): return self.repo.get_all(filters=filters) @@ -14,13 +13,3 @@ def list_reviews(self, filters=None): def get_review(self, review_id): return self.repo.get_by_id(review_id) - def poll_reviews(self, app_id, date_from=None, date_to=None): - reviews = self.adapter.fetch_reviews(app_id=app_id, date_from=date_from, date_to=date_to) - - saved_count = self.repo.save_reviews(app_id, reviews) - - return { - "fetched": len(reviews), - "saved": saved_count, - "reviews": reviews, - } diff --git a/review/urls.py b/review/urls.py index cec6f9c..f43b970 100644 --- a/review/urls.py +++ b/review/urls.py @@ -8,9 +8,4 @@ urlpatterns = [ path("", include(router.urls)), - path( - "apps//reviews/polling/", - ReviewViewSet.as_view({"post": "poll_reviews"}), - name="poll-reviews", - ), ] diff --git a/review/views.py b/review/views.py index 80a1d7c..505ddbf 100644 --- a/review/views.py +++ b/review/views.py @@ -37,45 +37,3 @@ def retrieve(self, request, pk=None): source = self.service.get_review(pk) serializer = ReviewSerializer(source) return Response(serializer.data) - - @extend_schema( - request=None, - parameters=[ - OpenApiParameter( - name="id", - type=int, - required=True, - location="path", - description="App ID to fetch reviews for", - ), - OpenApiParameter( - name="date_from", type=str, description="Start date (YYYY-MM-DD)", required=False - ), - OpenApiParameter( - name="date_to", type=str, description="End date (YYYY-MM-DD)", required=False - ), - ], - responses={ - 200: OpenApiResponse(description="Reviews successfully fetched and saved."), - 400: OpenApiResponse( - description="Invalid request. Possible causes: missing parameters, " - "wrong date format, or invalid date range." - ), - 404: OpenApiResponse(description="App with the given ID was not found."), - }, - tags=["Reviews"], - description=( - "Polls Google Play for reviews of a given app between two dates.\n\n" - "Saves new ones in the database and returns" - "the fetched reviews with stats.\n\n" - "If no `date_from` and `date_to` are provided," - " reviews from **yesterday** will be fetched." - ), - ) - def poll_reviews(self, request, pk=None): - date_from = request.query_params.get("date_from") - date_to = request.query_params.get("date_to") - - result = self.service.poll_reviews(app_id=int(pk), date_from=date_from, date_to=date_to) - - return Response(result) diff --git a/schema.yaml b/schema.yaml new file mode 100644 index 0000000..0fb9c0b --- /dev/null +++ b/schema.yaml @@ -0,0 +1,1237 @@ +openapi: 3.0.3 +info: + title: '' + version: 0.0.0 +paths: + /api/apps/: + get: + operationId: api_apps_list + tags: + - Apps + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/App' + description: '' + post: + operationId: api_apps_create + description: |- + Creates a new app. + + The fields 'available_on_ios', 'available_on_android', 'pegi_rating', 'release_date', and 'min_ios_version' will be automatically populated by fetching data from the App Store and/or Google Play APIs based on the provided identifiers. + + You do not need to manually provide these fields when creating an app. + tags: + - Apps + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/AppCreate' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/AppCreate' + multipart/form-data: + schema: + $ref: '#/components/schemas/AppCreate' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '201': + content: + application/json: + schema: + $ref: '#/components/schemas/App' + description: '' + /api/apps/{id}/: + get: + operationId: api_apps_retrieve + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Apps + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/App' + description: '' + put: + operationId: api_apps_update + parameters: + - in: path + name: id + schema: + type: string + required: true + tags: + - Apps + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/App' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/App' + multipart/form-data: + schema: + $ref: '#/components/schemas/App' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/App' + description: '' + delete: + operationId: api_apps_destroy + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Apps + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '204': + description: No response body + /api/metric-values/: + get: + operationId: api_metric_values_list + tags: + - Metrics + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/MetricValue' + description: '' + /api/metric-values/{id}/: + get: + operationId: api_metric_values_retrieve + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Metrics + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/MetricValue' + description: '' + /api/metrics/: + get: + operationId: api_metrics_list + tags: + - Metrics + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Metric' + description: '' + post: + operationId: api_metrics_create + description: |- + Creates a new data metric. + + The 'source' field is read-only and cannot be set directly on creation. + tags: + - Metrics + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Metric' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/Metric' + multipart/form-data: + schema: + $ref: '#/components/schemas/Metric' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '201': + content: + application/json: + schema: + $ref: '#/components/schemas/Metric' + description: '' + /api/metrics/{id}/: + get: + operationId: api_metrics_retrieve + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Metrics + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Metric' + description: '' + put: + operationId: api_metrics_update + parameters: + - in: path + name: id + schema: + type: string + required: true + tags: + - Metrics + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Metric' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/Metric' + multipart/form-data: + schema: + $ref: '#/components/schemas/Metric' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Metric' + description: '' + delete: + operationId: api_metrics_destroy + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Metrics + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '204': + description: No response body + /api/metrics/{id}/sources/: + post: + operationId: api_metrics_sources_create + description: Adds a list of source IDs to an existing metric without removing + existing ones. + parameters: + - in: path + name: id + schema: + type: string + required: true + tags: + - Metrics + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/LinkSource' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/LinkSource' + multipart/form-data: + schema: + $ref: '#/components/schemas/LinkSource' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Metric' + description: '' + delete: + operationId: api_metrics_sources_destroy + description: Removes a list of source IDs from an existing metric. + parameters: + - in: path + name: id + schema: + type: string + required: true + tags: + - Metrics + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '204': + description: No response body + /api/polling/metrics/: + get: + operationId: api_polling_metrics_list + description: List all active metric polling schedules. + tags: + - Polling + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/PollingSchedule' + description: '' + /api/polling/metrics/apps/{id}/: + get: + operationId: api_polling_metrics_apps_retrieve + description: Check if there is an active polling schedule for an app. + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Polling + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/PollingSchedule' + description: '' + post: + operationId: api_polling_metrics_apps_create + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - api + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PollingSchedule' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/PollingSchedule' + multipart/form-data: + schema: + $ref: '#/components/schemas/PollingSchedule' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '201': + content: + application/json: + schema: + $ref: '#/components/schemas/PollingSchedule' + description: '' + /api/polling/reviews/: + get: + operationId: api_polling_reviews_list + description: List all active metric polling schedules. + tags: + - Polling + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/PollingSchedule' + description: '' + /api/polling/reviews/apps/{id}/: + get: + operationId: api_polling_reviews_apps_retrieve + description: Check if there is an active polling schedule for an app. + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Polling + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/PollingSchedule' + description: '' + post: + operationId: api_polling_reviews_apps_create + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - api + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/PollingSchedule' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/PollingSchedule' + multipart/form-data: + schema: + $ref: '#/components/schemas/PollingSchedule' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '201': + content: + application/json: + schema: + $ref: '#/components/schemas/PollingSchedule' + description: '' + /api/reviews/: + get: + operationId: api_reviews_list + parameters: + - in: query + name: app + schema: + type: integer + description: Filter by app ID + - in: query + name: date_from + schema: + type: string + description: Start date (YYYY-MM-DD) + - in: query + name: date_to + schema: + type: string + description: End date (YYYY-MM-DD) + tags: + - Reviews + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Review' + description: '' + /api/reviews/{id}/: + get: + operationId: api_reviews_retrieve + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Reviews + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Review' + description: '' + /api/schema/: + get: + operationId: api_schema_retrieve + description: |- + OpenApi3 schema for this API. Format can be selected via content negotiation. + + - YAML: application/vnd.oai.openapi + - JSON: application/vnd.oai.openapi+json + parameters: + - in: query + name: format + schema: + type: string + enum: + - json + - yaml + - in: query + name: lang + schema: + type: string + enum: + - af + - ar + - ar-dz + - ast + - az + - be + - bg + - bn + - br + - bs + - ca + - ckb + - cs + - cy + - da + - de + - dsb + - el + - en + - en-au + - en-gb + - eo + - es + - es-ar + - es-co + - es-mx + - es-ni + - es-ve + - et + - eu + - fa + - fi + - fr + - fy + - ga + - gd + - gl + - he + - hi + - hr + - hsb + - hu + - hy + - ia + - id + - ig + - io + - is + - it + - ja + - ka + - kab + - kk + - km + - kn + - ko + - ky + - lb + - lt + - lv + - mk + - ml + - mn + - mr + - ms + - my + - nb + - ne + - nl + - nn + - os + - pa + - pl + - pt + - pt-br + - ro + - ru + - sk + - sl + - sq + - sr + - sr-latn + - sv + - sw + - ta + - te + - tg + - th + - tk + - tr + - tt + - udm + - ug + - uk + - ur + - uz + - vi + - zh-hans + - zh-hant + tags: + - api + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/vnd.oai.openapi: + schema: + type: object + additionalProperties: {} + application/yaml: + schema: + type: object + additionalProperties: {} + application/vnd.oai.openapi+json: + schema: + type: object + additionalProperties: {} + application/json: + schema: + type: object + additionalProperties: {} + description: '' + /api/sources/: + get: + operationId: api_sources_list + tags: + - Sources + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Source' + description: '' + post: + operationId: api_sources_create + description: |- + Creates a new data source. + + The 'metrics' field is read-only and cannot be set directly on creation. + tags: + - Sources + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Source' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/Source' + multipart/form-data: + schema: + $ref: '#/components/schemas/Source' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '201': + content: + application/json: + schema: + $ref: '#/components/schemas/Source' + description: '' + /api/sources/{id}/: + get: + operationId: api_sources_retrieve + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Sources + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Source' + description: '' + put: + operationId: api_sources_update + parameters: + - in: path + name: id + schema: + type: string + required: true + tags: + - Sources + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Source' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/Source' + multipart/form-data: + schema: + $ref: '#/components/schemas/Source' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Source' + description: '' + delete: + operationId: api_sources_destroy + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Sources + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '204': + description: No response body + /api/sources/{id}/metrics/: + post: + operationId: api_sources_metrics_create + description: Adds a list of metric IDs to an existing source without removing + existing ones. + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Sources + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/LinkMetrics' + application/x-www-form-urlencoded: + schema: + $ref: '#/components/schemas/LinkMetrics' + multipart/form-data: + schema: + $ref: '#/components/schemas/LinkMetrics' + required: true + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '200': + content: + application/json: + schema: + $ref: '#/components/schemas/Source' + description: '' + delete: + operationId: api_sources_metrics_destroy + description: Removes a list of metric IDs from an existing source. + parameters: + - in: path + name: id + schema: + type: integer + required: true + tags: + - Sources + security: + - cookieAuth: [] + - basicAuth: [] + - {} + responses: + '204': + description: No response body +components: + schemas: + App: + type: object + properties: + id: + type: integer + readOnly: true + code: + type: string + maxLength: 100 + name: + type: string + maxLength: 200 + description: + type: string + appstore_id: + type: string + nullable: true + maxLength: 100 + playstore_id: + type: string + nullable: true + maxLength: 100 + developer: + type: string + nullable: true + maxLength: 100 + available_on_ios: + type: boolean + available_on_android: + type: boolean + pegi_rating: + type: string + nullable: true + maxLength: 10 + release_date: + type: string + format: date + nullable: true + min_ios_version: + type: string + nullable: true + maxLength: 10 + required: + - code + - description + - id + - name + AppCreate: + type: object + properties: + code: + type: string + maxLength: 100 + name: + type: string + maxLength: 200 + description: + type: string + appstore_id: + type: string + nullable: true + maxLength: 100 + playstore_id: + type: string + nullable: true + maxLength: 100 + required: + - code + - description + - name + LinkMetrics: + type: object + properties: + metrics: + type: array + items: + type: integer + description: List of metric IDs to link to the source. + required: + - metrics + LinkSource: + type: object + properties: + sources: + type: array + items: + type: integer + description: List of source IDs to link to the metric. + required: + - sources + Metric: + type: object + properties: + id: + type: integer + readOnly: true + code: + type: string + maxLength: 100 + name: + type: string + maxLength: 100 + description: + type: string + value_type: + $ref: '#/components/schemas/ValueTypeEnum' + sources: + type: array + items: + type: integer + readOnly: true + required: + - code + - description + - id + - name + - sources + - value_type + MetricValue: + type: object + properties: + id: + type: integer + readOnly: true + app: + type: integer + readOnly: true + metric: + type: integer + readOnly: true + source: + type: integer + readOnly: true + nullable: true + value: + type: string + readOnly: true + retrieved_at: + type: string + format: date-time + readOnly: true + required: + - app + - id + - metric + - retrieved_at + - source + - value + PolarityEnum: + enum: + - positive + - neutral + - negative + type: string + description: |- + * `positive` - Positive + * `neutral` - Neutral + * `negative` - Negative + PollTypeEnum: + enum: + - metrics + - reviews + - both + type: string + description: |- + * `metrics` - Metrics only + * `reviews` - Reviews only + * `both` - Metrics + Reviews + PollingSchedule: + type: object + properties: + app: + type: integer + interval_hours: + type: integer + maximum: 9223372036854775807 + minimum: -9223372036854775808 + format: int64 + start_at: + type: string + format: date-time + readOnly: true + next_run: + type: string + format: date-time + readOnly: true + poll_type: + allOf: + - $ref: '#/components/schemas/PollTypeEnum' + default: metrics + is_active: + type: boolean + readOnly: true + required: + - app + - interval_hours + - is_active + - next_run + - start_at + Review: + type: object + properties: + id: + type: integer + readOnly: true + review_id: + type: string + readOnly: true + nullable: true + app: + type: integer + readOnly: true + author: + type: string + readOnly: true + nullable: true + content: + type: string + readOnly: true + rating: + type: number + format: double + readOnly: true + nullable: true + polarity: + $ref: '#/components/schemas/PolarityEnum' + type: + $ref: '#/components/schemas/ReviewTypeEnum' + topic: + $ref: '#/components/schemas/TopicEnum' + date: + type: string + format: date-time + readOnly: true + reply_content: + type: string + readOnly: true + nullable: true + replied_at: + type: string + format: date-time + readOnly: true + nullable: true + app_version: + type: string + readOnly: true + nullable: true + required: + - app + - app_version + - author + - content + - date + - id + - polarity + - rating + - replied_at + - reply_content + - review_id + - topic + - type + ReviewTypeEnum: + enum: + - bug + - feature + - general + type: string + description: |- + * `bug` - Bug + * `feature` - Feature + * `general` - General + Source: + type: object + properties: + id: + type: integer + readOnly: true + code: + type: string + maxLength: 100 + name: + type: string + maxLength: 100 + type: + allOf: + - $ref: '#/components/schemas/SourceTypeEnum' + description: |- + Type of the source. Available options: + - 'api': Data source accessible through a public API. + - 'scraper': Data source obtained via web scraping. + - 'external_tool': Data provided by an external tool. + + * `api` - API + * `scraper` - Scraper + * `external_tool` - External Tool + url: + type: string + format: uri + nullable: true + maxLength: 200 + metrics: + type: array + items: + type: integer + readOnly: true + required: + - code + - id + - metrics + - name + - type + SourceTypeEnum: + enum: + - api + - scraper + - external_tool + type: string + description: |- + * `api` - API + * `scraper` - Scraper + * `external_tool` - External Tool + TopicEnum: + enum: + - aesthetics + - compatibility + - cost + - effectiveness + - efficiency + - enjoyability + - learnability + - security + - usability + type: string + description: |- + * `aesthetics` - Aesthetics + * `compatibility` - Compatibility + * `cost` - Cost + * `effectiveness` - Effectiveness + * `efficiency` - Efficiency + * `enjoyability` - Enjoyability + * `learnability` - Learnability + * `security` - Security + * `usability` - Usability + ValueTypeEnum: + enum: + - string + - integer + - float + - date + - boolean + type: string + description: |- + * `string` - String + * `integer` - Integer + * `float` - Float + * `date` - Date + * `boolean` - Boolean + securitySchemes: + basicAuth: + type: http + scheme: basic + cookieAuth: + type: apiKey + in: cookie + name: sessionid From 3004fc670d8d081690654b80c2b99c867b5d21b5 Mon Sep 17 00:00:00 2001 From: Anyerrr Date: Mon, 5 May 2025 11:23:31 +0200 Subject: [PATCH 35/42] =?UTF-8?q?feat(celery):=20Configuraci=C3=B3=20corre?= =?UTF-8?q?cte=20de=20docker=20compose?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/settings.py | 4 ++-- docker-compose.yml | 2 -- polling/tasks.py | 10 +++++++++- requirements.txt | 1 + 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/config/settings.py b/config/settings.py index dfe1c00..2081262 100644 --- a/config/settings.py +++ b/config/settings.py @@ -141,5 +141,5 @@ "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", } -CELERY_BROKER_URL = 'redis://localhost:6379/0' -CELERY_RESULT_BACKEND = 'redis://localhost:6379/0' +CELERY_BROKER_URL = 'redis://redis:6379/0' +CELERY_RESULT_BACKEND = 'redis://redis:6379/0' diff --git a/docker-compose.yml b/docker-compose.yml index fa3bbbd..8cff519 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - services: redis: image: redis:7 diff --git a/polling/tasks.py b/polling/tasks.py index 235f551..e278033 100644 --- a/polling/tasks.py +++ b/polling/tasks.py @@ -1,4 +1,5 @@ from celery import shared_task +import time @shared_task def run_polling_task(app_id, poll_type): @@ -7,4 +8,11 @@ def run_polling_task(app_id, poll_type): if poll_type == "metrics": result = service.poll_metrics(app_id) elif poll_type == "reviews": - result = service.poll_reviews(app_id) \ No newline at end of file + result = service.poll_reviews(app_id) + +@shared_task +def prueba_celery(): + print("Executant tasca de prova...") + time.sleep(5) + print("Tasca completada correctament!") + return "Fet!" diff --git a/requirements.txt b/requirements.txt index a982cca..e5defd0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,6 +46,7 @@ python-crontab==3.2.0 python-dateutil==2.9.0.post0 python-dotenv==1.1.0 PyYAML==6.0.2 +redis==6.0.0 referencing==0.36.2 requests==2.32.3 rpds-py==0.24.0 From 4508663db589ac62405230a31f995503d772e165 Mon Sep 17 00:00:00 2001 From: Anyerrr Date: Mon, 5 May 2025 11:37:19 +0200 Subject: [PATCH 36/42] refact(readme): canvi en el readme i creada la license --- LICENSE | 21 +++++++++++ README.md | 109 +++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 113 insertions(+), 17 deletions(-) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f4586ac --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 [Anyer Moreno Alcaraz] + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 87d183b..ce07ad6 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,106 @@ # MetaAppCollector - Backend -MetaAppCollector és una eina per agregar metadades d'aplicacions mòbils des de diverses fonts externes, com botigues d'apps, xarxes socials i motors de cerca. +**MetaAppCollector** és una eina *open source* per monitoritzar aplicacions mòbils agregant informació pública de diverses fonts com botigues d'apps i xarxes socials. -Aquest projecte és el backend del sistema, desenvolupat amb **Django**. +> 🧪 Desenvolupada amb Django, Celery, Redis i Docker. Pensada per a desenvolupadors, investigadors i equips de màrqueting que volen entendre millor la visibilitat de les seves apps. -## 🔧 Requisits +![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg) -- Python 3.12 -- Django 5.1.7 -- Git +--- -## 🚀 Instal·lació +## 🔧 Funcionalitats principals + +- 🔍 Recuperació de metadades i reviews d'aplicacions mòbils (Google Play, etc.) +- 📊 Agregació de mètriques d'interacció i visibilitat +- ⏱️ Polling periòdic configurable per recollida automàtica +- 🧠 Arquitectura extensible amb adaptadors dinàmics +- 🐇 Execució asíncrona de tasques amb Celery + Redis + +--- + +## 🚀 Posada en marxa amb Docker + +Assegura’t de tenir instal·lats: + +- [Docker](https://www.docker.com/) +- [Docker Compose](https://docs.docker.com/compose/) + +### 1. Clona el projecte + +```bash +git clone https://github.com/el-teu-usuari/metaappcollector-backend.git +cd metaappcollector-backend +``` + +### 2. Configura variables d'entorn + +Crea un fitxer .env a l'arrel amb el següent contingut mínim: + +```bash +DJANGO_SECRET_KEY=your-secret-key +DJANGO_DEBUG=True +``` + +### 3. Executa els serveis ```bash -git clone git@github.com:Anyerrr/MetaAppCollector-Backend.git -cd MetaAppCollector-Backend -python -m venv .venv -.venv\Scripts\activate -pip install -r requirements.txt +docker compose up --build ``` +Això iniciarà: -## 🛠 ️ Execució +- El servidor Django a http://localhost:8000 +- El worker de Celery +- El Beat de Celery (scheduler) +- Redis com a broker + +## 🧪 Prova Celery +Obre una shell dins del contenidor web: +```bash +docker compose exec web python manage.py shell +``` +I prova una tasca: +```bash +from polling.tasks import prueba_celery +prueba_celery.delay() +``` +Pots veure els resultats al log del worker: +```bash +docker compose logs -f celery +``` + +## 🧱 Estructura del projecte ```bash -python manage.py runserver +├── polling/ # Lògica de polling i tasques Celery +├── metric/ # Agregació de mètriques +├── review/ # Gestió de reviews +├── source/ # Fonts de dades integrades +├── app/ # Aplicacions mòbils registrades +├── config/ # Configuració general de Django i Celery +├── docker-compose.yml # Definició dels serveis ``` -## 🧪 Proves +## 📚 Comandes útils ```bash -python manage.py test -``` \ No newline at end of file +docker compose exec web python manage.py migrate # Aplicar migracions +docker compose exec web python manage.py createsuperuser +docker compose exec web python manage.py shell # Obrir shell +``` + +## 🤝 Contribució +Les contribucions són benvingudes! Si vols col·laborar: + +1. Fes un fork del repositori +2. Crea una branca nova (git checkout -b feature/nou-adaptador) +3. Fes els teus canvis i commiteja (git commit -am 'Afegit adaptador per X') +4. Puja la branca (git push origin feature/nou-adaptador) +5. Fes una pull request 🙌 + +Abans d’enviar, assegura’t de seguir l’estil de codi del projecte i escriure tests quan sigui necessari. + + +## ✉️ Contacte +Per dubtes tècnics o col·laboració, contacta amb el desenvolupador principal: +📧 a.moreno@estudiantat.upc.edu + +## 📄 Llicència +Aquest projecte està llicenciat sota la MIT License – consulta el fitxer LICENSE per a més informació. \ No newline at end of file From a816820e0dc2b1064833d0aa6740b5370f3ca530 Mon Sep 17 00:00:00 2001 From: Anyerrr Date: Mon, 5 May 2025 14:04:46 +0200 Subject: [PATCH 37/42] feat(celery): Acabar de polir els processos --- .../0002_pollingschedule_periodic_task.py | 26 ++++++++++++++++ polling/models.py | 13 +++++++- polling/repositories.py | 27 +++++++++++----- polling/serializers.py | 31 +++++++++++++++++-- polling/services.py | 13 +++++--- polling/tasks.py | 9 ++++-- polling/urls.py | 5 ++- polling/views.py | 25 +++++++-------- source/adapters/base.py | 3 ++ source/adapters/google_play_scraper.py | 2 +- 10 files changed, 121 insertions(+), 33 deletions(-) create mode 100644 polling/migrations/0002_pollingschedule_periodic_task.py diff --git a/polling/migrations/0002_pollingschedule_periodic_task.py b/polling/migrations/0002_pollingschedule_periodic_task.py new file mode 100644 index 0000000..4389c85 --- /dev/null +++ b/polling/migrations/0002_pollingschedule_periodic_task.py @@ -0,0 +1,26 @@ +# Generated by Django 5.1.7 on 2025-05-05 10:20 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("django_celery_beat", "0019_alter_periodictasks_options"), + ("polling", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="pollingschedule", + name="periodic_task", + field=models.OneToOneField( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="polling_schedule", + to="django_celery_beat.periodictask", + ), + ), + ] diff --git a/polling/models.py b/polling/models.py index 787cff3..32bfcb5 100644 --- a/polling/models.py +++ b/polling/models.py @@ -1,5 +1,5 @@ from django.db import models - +from django_celery_beat.models import PeriodicTask from app.models import App @@ -18,5 +18,16 @@ class PollingSchedule(models.Model): is_active = models.BooleanField(default=True) + periodic_task = models.OneToOneField( + PeriodicTask, + on_delete=models.CASCADE, + null=True, + blank=True, + related_name="polling_schedule" + ) + class Meta: unique_together = ("app", "poll_type") + + def __str__(self): + return f"{self.app.name} - {self.poll_type}" diff --git a/polling/repositories.py b/polling/repositories.py index b24d46c..e68816a 100644 --- a/polling/repositories.py +++ b/polling/repositories.py @@ -1,3 +1,5 @@ +from django.shortcuts import get_object_or_404 + from .models import PollingSchedule from django_celery_beat.models import PeriodicTask, IntervalSchedule import json @@ -12,13 +14,10 @@ def get_all_active(poll_type): @staticmethod def get_by_app_id(app_id, poll_type): - queryset = PollingSchedule.objects.filter(app_id=app_id, is_active=True) - if poll_type: - queryset = queryset.filter(poll_type=poll_type) - return queryset.first() + return get_object_or_404(PollingSchedule, app_id=app_id, poll_type=poll_type, is_active=True) @staticmethod - def create(app_id, interval_hours, poll_type): + def create(app_id, interval_hours, poll_type, periodic_task): from django.utils.timezone import now from datetime import timedelta @@ -30,6 +29,7 @@ def create(app_id, interval_hours, poll_type): next_run=start + timedelta(hours=interval_hours), poll_type=poll_type, is_active=True, + periodic_task=periodic_task, ) return schedule @@ -43,9 +43,20 @@ def get_or_create_interval_schedule(interval_hours): @staticmethod def create_periodic_task(schedule, app_id, poll_type, task): - PeriodicTask.objects.create( + name = f"{poll_type}-polling-{app_id}" + if PeriodicTask.objects.filter(name=name).exists(): + raise ValueError("PeriodicTask already exists with this name") + + task = PeriodicTask.objects.create( interval=schedule, - name=f"{poll_type}-polling-{app_id}", + name=name, task=task, args=json.dumps([app_id, poll_type]), - ) \ No newline at end of file + ) + return task + + @staticmethod + def delete(instance): + if instance.periodic_task: + instance.periodic_task.delete() + instance.delete() \ No newline at end of file diff --git a/polling/serializers.py b/polling/serializers.py index fd5510e..f4a9313 100644 --- a/polling/serializers.py +++ b/polling/serializers.py @@ -1,10 +1,37 @@ from rest_framework import serializers from polling.models import PollingSchedule +from django_celery_beat.models import IntervalSchedule, PeriodicTask +class IntervalScheduleSerializer(serializers.ModelSerializer): + class Meta: + model = IntervalSchedule + fields = ["id", "every", "period"] + +class PeriodicTaskSerializer(serializers.ModelSerializer): + interval = IntervalScheduleSerializer(read_only=True) + + class Meta: + model = PeriodicTask + fields = ["id", "name", "task", "args", "enabled", "last_run_at", "interval"] class PollingScheduleSerializer(serializers.ModelSerializer): + periodic_task = PeriodicTaskSerializer(read_only=True) + class Meta: model = PollingSchedule - fields = ["app", "interval_hours", "start_at", "next_run", "is_active"] - read_only_fields = ["app", "start_at", "next_run", "is_active"] + fields = [ + "app", + "interval_hours", + "start_at", + "next_run", + "is_active", + "periodic_task", + ] + read_only_fields = [ + "app", + "start_at", + "next_run", + "is_active", + "periodic_task", + ] diff --git a/polling/services.py b/polling/services.py index b2fcb02..8779934 100644 --- a/polling/services.py +++ b/polling/services.py @@ -39,13 +39,18 @@ def create_polling_schedule(self, app_id, interval_hours=None, poll_type=None): if existing: raise ValueError(f"Polling already scheduled for app {app_id} and type {poll_type}") - schedule = self.repo.create_polling_schedule(app_id,interval_hours,poll_type) + interval_schedule = self.repo.get_or_create_interval_schedule(interval_hours) + periodic_task = self.repo.create_periodic_task(interval_schedule, app_id, poll_type, "polling.tasks.run_polling_task") + schedule = self.repo.create(app_id, interval_hours, poll_type, periodic_task) - schedule2 = self.repo.get_or_create_interval_schedule(interval_hours) - - self.repo.create_perdioc_task(schedule2,app_id,poll_type,"polling.tasks.run_polling_task") + run_polling_task.delay(app_id, poll_type) return schedule + def delete_schedule(self, instance): + return self.repo.delete(instance) + def poll_metrics(self, app_id): ... + + diff --git a/polling/tasks.py b/polling/tasks.py index e278033..47d03e5 100644 --- a/polling/tasks.py +++ b/polling/tasks.py @@ -5,10 +5,15 @@ def run_polling_task(app_id, poll_type): from polling.services import PollingService service = PollingService() + if poll_type == "metrics": - result = service.poll_metrics(app_id) + ##return service.poll_metrics(app_id) + print("poll_metrics completat") elif poll_type == "reviews": - result = service.poll_reviews(app_id) + ##return service.poll_reviews(app_id + print("poll_reviews completat") + else: + raise ValueError(f"Unknown poll_type: {poll_type}") @shared_task def prueba_celery(): diff --git a/polling/urls.py b/polling/urls.py index 335ba1a..f62dcd9 100644 --- a/polling/urls.py +++ b/polling/urls.py @@ -6,12 +6,15 @@ polling_list_view = PollingScheduleViewSet.as_view({"get": "list"}) polling_detail_view = PollingScheduleViewSet.as_view({ "get": "retrieve", - "post": "create" + "post": "create", + "delete": "destroy" }) +polling_review_view = PollingScheduleViewSet.as_view({"post": "poll_reviews"}) urlpatterns = [ path("metrics/", polling_list_view, {"poll_type": "metrics"}), path("reviews/", polling_list_view, {"poll_type": "reviews"}), path("metrics/apps//", polling_detail_view, {"poll_type": "metrics"}), path("reviews/apps//", polling_detail_view, {"poll_type": "reviews"}), + path("maunal/reviews/apps//", polling_review_view), ] diff --git a/polling/views.py b/polling/views.py index 4ce3cde..94bdfe2 100644 --- a/polling/views.py +++ b/polling/views.py @@ -32,15 +32,6 @@ def list(self, request, poll_type=None): ) def retrieve(self, request, id=None, poll_type=None): schedule = self.service.get_polling_schedule(id, poll_type) - if not schedule: - if poll_type == "metrics": - message = f"There is no active polling schedule for metrics for app {id}." - elif poll_type == "reviews": - message = f"There is no active polling schedule for reviews for app {id}." - else: - message = f"No active polling schedule found for app {id}." - return Response({"detail": message}, status=status.HTTP_404_NOT_FOUND) - serializer = PollingScheduleSerializer(schedule) return Response(serializer.data, status=status.HTTP_200_OK) @@ -54,17 +45,23 @@ def create(self, request, id=None, poll_type=None): serializer.is_valid(raise_exception=True) interval = serializer.validated_data["interval_hours"] - app = AppService().get_app(id) - if not app: - return Response({"detail": "App not found"}, status=404) - try: - schedule = self.service.create_polling_schedule(app, interval, poll_type) + schedule = self.service.create_polling_schedule(id, interval, poll_type) except ValueError as e: return Response({"detail": str(e)}, status=409) return Response(PollingScheduleSerializer(schedule).data, status=201) + @extend_schema( + parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], + responses={204: None}, + tags=["Polling"], + ) + def destroy(self, request, id=None, poll_type=None): + schedule = self.service.get_polling_schedule(id, poll_type) + self.service.delete_schedule(schedule) + return Response(status=status.HTTP_204_NO_CONTENT) + @extend_schema( request=None, parameters=[ diff --git a/source/adapters/base.py b/source/adapters/base.py index 52d27b8..8f23139 100644 --- a/source/adapters/base.py +++ b/source/adapters/base.py @@ -21,3 +21,6 @@ def supports_metric(self, metric: str) -> bool: ... @abstractmethod def fetch(self, app_id: int, metrics: list[str]) -> dict[str, str]: ... + + def fetch_reviews(self, app_id: int, date_from=None, date_to=None): + return [] \ No newline at end of file diff --git a/source/adapters/google_play_scraper.py b/source/adapters/google_play_scraper.py index 981d37c..7797638 100644 --- a/source/adapters/google_play_scraper.py +++ b/source/adapters/google_play_scraper.py @@ -80,7 +80,7 @@ def fetch_reviews(self, app_id: int, date_from=None, date_to=None): app.playstore_id, lang="en", country="us", - count=count_per_request, + count=10, continuation_token=next_token, ) count += 1 From aedff289e56dc961ea21bd13540eb51c5b4fca99 Mon Sep 17 00:00:00 2001 From: Anyerrr Date: Mon, 5 May 2025 16:11:27 +0200 Subject: [PATCH 38/42] feat(poll_reviews): poll_reviews manual acabat --- app/repositories.py | 4 +--- app/services.py | 8 +++++++- metric/repositories.py | 6 ++---- metric/services.py | 15 +++++++++++---- review/repositories.py | 3 +-- review/services.py | 9 ++++++++- source/adapters/reddit.py | 21 ++++++++++++++------- source/repositories.py | 4 +--- source/services.py | 13 +++++++++++-- 9 files changed, 56 insertions(+), 27 deletions(-) diff --git a/app/repositories.py b/app/repositories.py index 1b47aae..0a5d2db 100644 --- a/app/repositories.py +++ b/app/repositories.py @@ -1,5 +1,3 @@ -from django.shortcuts import get_object_or_404 - from .models import App @@ -10,7 +8,7 @@ def get_all(): @staticmethod def get_by_id(app_id): - return get_object_or_404(App, id=app_id) + return App.objects.get(id=app_id) @staticmethod def create(data): diff --git a/app/services.py b/app/services.py index 02a035b..4e81cfc 100644 --- a/app/services.py +++ b/app/services.py @@ -1,4 +1,6 @@ from datetime import datetime +from rest_framework.exceptions import NotFound +from django.core.exceptions import ObjectDoesNotExist from .repositories import AppRepository @@ -29,7 +31,11 @@ def list_apps(self): return self.repo.get_all() def get_app(self, app_id): - return self.repo.get_by_id(app_id) + try: + return self.repo.get_by_id(app_id) + except ObjectDoesNotExist: + raise NotFound(f"The app with ID '{app_id}' is not registered.") + def create_app(self, validated_data): self._fetch_appstore_data(validated_data) diff --git a/metric/repositories.py b/metric/repositories.py index a132a18..60e31c7 100644 --- a/metric/repositories.py +++ b/metric/repositories.py @@ -1,5 +1,3 @@ -from django.shortcuts import get_object_or_404 - from .models import Metric, MetricValue @@ -10,7 +8,7 @@ def get_all(): @staticmethod def get_by_id(pk): - return get_object_or_404(Metric, id=pk) + return Metric.objects.get(id=pk) @staticmethod def create(data): @@ -47,7 +45,7 @@ def get_all(): @staticmethod def get_by_id(pk): - return get_object_or_404(MetricValue, id=pk) + return MetricValue.objects.get(id=pk) @staticmethod def create(data): diff --git a/metric/services.py b/metric/services.py index c7dda3b..d692d6c 100644 --- a/metric/services.py +++ b/metric/services.py @@ -1,5 +1,6 @@ from .repositories import MetricRepository, MetricValueRepository - +from rest_framework.exceptions import NotFound +from django.core.exceptions import ObjectDoesNotExist class MetricService: def __init__(self): @@ -9,7 +10,10 @@ def list_metrics(self): return self.repo.get_all() def get_metric(self, pk): - return self.repo.get_by_id(pk) + try: + return self.repo.get_by_id(pk) + except ObjectDoesNotExist: + raise NotFound(f"The metric with ID '{pk}' is not registered.") def create_metric(self, validated_data): return self.repo.create(validated_data) @@ -35,8 +39,11 @@ def list_metric_values(self): return self.repo.get_all() def get_metric_value(self, pk): - return self.repo.get_by_id(pk) - + try: + return self.repo.get_by_id(pk) + except ObjectDoesNotExist: + raise NotFound(f"The metric_value with ID '{pk}' is not registered.") + def create_metric_value(self, validated_data): return self.repo.create(validated_data) diff --git a/review/repositories.py b/review/repositories.py index 21b8ff0..560cce6 100644 --- a/review/repositories.py +++ b/review/repositories.py @@ -1,6 +1,5 @@ from datetime import datetime -from django.shortcuts import get_object_or_404 from django.utils import timezone from .models import Review @@ -33,7 +32,7 @@ def get_all(self, filters=None): return queryset def get_by_id(self, review_id): - return get_object_or_404(Review, id=review_id) + return Review.objects.get(id=review_id) def save_reviews(self, app_id, reviews_list): if not reviews_list: diff --git a/review/services.py b/review/services.py index be1520f..ec4e85c 100644 --- a/review/services.py +++ b/review/services.py @@ -1,3 +1,6 @@ +from rest_framework.exceptions import NotFound +from django.core.exceptions import ObjectDoesNotExist + from source.adapters.google_play_scraper import GooglePlayScraperAdapter from .repositories import ReviewRepository @@ -11,5 +14,9 @@ def list_reviews(self, filters=None): return self.repo.get_all(filters=filters) def get_review(self, review_id): - return self.repo.get_by_id(review_id) + try: + return self.repo.get_by_id(review_id) + except ObjectDoesNotExist: + raise NotFound(f"The review with ID '{review_id}' is not registered.") + diff --git a/source/adapters/reddit.py b/source/adapters/reddit.py index 9934d07..4eb1a77 100644 --- a/source/adapters/reddit.py +++ b/source/adapters/reddit.py @@ -2,6 +2,8 @@ import praw +from rest_framework.exceptions import APIException + from app.services import AppService from metric.constants import MetricCodes from source.adapters.base import SourceAdapter @@ -18,13 +20,18 @@ def __init__(self): self.type = source_data["type"] self.url = source_data["url"] self.supported_metrics = source_data["supported_metrics"] - self.reddit = praw.Reddit( - client_id=os.getenv("REDDIT_CLIENT_ID"), - client_secret=os.getenv("REDDIT_CLIENT_SECRET"), - user_agent=os.getenv("REDDIT_USER_AGENT"), - username=os.getenv("REDDIT_USERNAME"), - password=os.getenv("REDDIT_PASSWORD"), - ) + try: + self.reddit = praw.Reddit( + client_id=os.getenv("REDDIT_CLIENT_ID"), + client_secret=os.getenv("REDDIT_CLIENT_SECRET"), + user_agent=os.getenv("REDDIT_USER_AGENT"), + username=os.getenv("REDDIT_USERNAME"), + password=os.getenv("REDDIT_PASSWORD"), + ) + except KeyError as e: + raise APIException(f"Reddit adapter initialization failed: missing env var {str(e)}.") + except praw.exceptions.MissingRequiredAttributeException as e: + raise APIException(f"Reddit adapter configuration error: {str(e)}") def supports_metric(self, metric: str) -> bool: return metric in self.supported_metrics diff --git a/source/repositories.py b/source/repositories.py index ffe8610..4ea641b 100644 --- a/source/repositories.py +++ b/source/repositories.py @@ -1,5 +1,3 @@ -from django.shortcuts import get_object_or_404 - from .models import Source @@ -11,7 +9,7 @@ def get_all(): @staticmethod def get_by_id(app_id): - return get_object_or_404(Source, id=app_id) + return Source.objects.get(id=app_id) @staticmethod def create(data): diff --git a/source/services.py b/source/services.py index d9fbd9c..4ccdf38 100644 --- a/source/services.py +++ b/source/services.py @@ -1,4 +1,6 @@ from source.adapters.base import SourceAdapter +from rest_framework.exceptions import NotFound +from django.core.exceptions import ObjectDoesNotExist from .repositories import SourceRepository @@ -11,7 +13,10 @@ def list_sources(self): return self.repo.get_all() def get_source(self, source_id): - return self.repo.get_by_id(source_id) + try: + return self.repo.get_by_id(source_id) + except ObjectDoesNotExist: + raise NotFound(f"The source with ID '{source_id}' is not registered.") def create_source(self, validated_data): return self.repo.create(validated_data) @@ -29,7 +34,11 @@ def remove_metrics(self, instance, metrics_ids): return self.repo.remove_metrics(instance, metrics_ids) def get_source_data(self, code: str) -> dict: - source = self.repo.get_by_code(code) + try: + source = self.repo.get_by_code(code) + except ObjectDoesNotExist: + raise NotFound(f"The source with code '{code}' is not registered.") + return { "code": source.code, "name": source.name, From b9ca752234d72b8d52b5b7143b50d8bd926088fc Mon Sep 17 00:00:00 2001 From: Anyer Date: Mon, 5 May 2025 23:25:11 +0200 Subject: [PATCH 39/42] =?UTF-8?q?feature(poll=5Fmetrics):=20Implementar=20?= =?UTF-8?q?funci=C3=B3=20per=20a=20la=20recollida=20peri=C3=B2dica=20de=20?= =?UTF-8?q?m=C3=A8triques?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services.py | 4 +- config/__init__.py | 2 +- config/celery.py | 7 +- config/settings.py | 4 +- metric/services.py | 8 +- polling/models.py | 3 +- polling/repositories.py | 26 +++--- polling/serializers.py | 5 +- polling/services.py | 119 +++++++++++++++++++------ polling/tasks.py | 10 ++- polling/urls.py | 8 +- polling/views.py | 8 +- review/services.py | 6 +- review/views.py | 2 +- source/adapters/base.py | 2 +- source/adapters/google_play_scraper.py | 2 +- source/adapters/reddit.py | 1 - source/services.py | 5 +- 18 files changed, 148 insertions(+), 74 deletions(-) diff --git a/app/services.py b/app/services.py index 4e81cfc..11b684a 100644 --- a/app/services.py +++ b/app/services.py @@ -1,6 +1,7 @@ from datetime import datetime -from rest_framework.exceptions import NotFound + from django.core.exceptions import ObjectDoesNotExist +from rest_framework.exceptions import NotFound from .repositories import AppRepository @@ -35,7 +36,6 @@ def get_app(self, app_id): return self.repo.get_by_id(app_id) except ObjectDoesNotExist: raise NotFound(f"The app with ID '{app_id}' is not registered.") - def create_app(self, validated_data): self._fetch_appstore_data(validated_data) diff --git a/config/__init__.py b/config/__init__.py index e31568a..53f4ccb 100644 --- a/config/__init__.py +++ b/config/__init__.py @@ -1,3 +1,3 @@ from .celery import app as celery_app -__all__ = ("celery_app",) \ No newline at end of file +__all__ = ("celery_app",) diff --git a/config/celery.py b/config/celery.py index d33bb1a..60888ac 100644 --- a/config/celery.py +++ b/config/celery.py @@ -1,8 +1,9 @@ import os + from celery import Celery -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings") -app = Celery('metaappcollector') -app.config_from_object('django.conf:settings', namespace='CELERY') +app = Celery("metaappcollector") +app.config_from_object("django.conf:settings", namespace="CELERY") app.autodiscover_tasks() diff --git a/config/settings.py b/config/settings.py index 2081262..9c04fe3 100644 --- a/config/settings.py +++ b/config/settings.py @@ -141,5 +141,5 @@ "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", } -CELERY_BROKER_URL = 'redis://redis:6379/0' -CELERY_RESULT_BACKEND = 'redis://redis:6379/0' +CELERY_BROKER_URL = "redis://redis:6379/0" +CELERY_RESULT_BACKEND = "redis://redis:6379/0" diff --git a/metric/services.py b/metric/services.py index d692d6c..80616df 100644 --- a/metric/services.py +++ b/metric/services.py @@ -1,6 +1,8 @@ -from .repositories import MetricRepository, MetricValueRepository -from rest_framework.exceptions import NotFound from django.core.exceptions import ObjectDoesNotExist +from rest_framework.exceptions import NotFound + +from .repositories import MetricRepository, MetricValueRepository + class MetricService: def __init__(self): @@ -43,7 +45,7 @@ def get_metric_value(self, pk): return self.repo.get_by_id(pk) except ObjectDoesNotExist: raise NotFound(f"The metric_value with ID '{pk}' is not registered.") - + def create_metric_value(self, validated_data): return self.repo.create(validated_data) diff --git a/polling/models.py b/polling/models.py index 32bfcb5..7bbec20 100644 --- a/polling/models.py +++ b/polling/models.py @@ -1,5 +1,6 @@ from django.db import models from django_celery_beat.models import PeriodicTask + from app.models import App @@ -23,7 +24,7 @@ class PollingSchedule(models.Model): on_delete=models.CASCADE, null=True, blank=True, - related_name="polling_schedule" + related_name="polling_schedule", ) class Meta: diff --git a/polling/repositories.py b/polling/repositories.py index e68816a..429ac1d 100644 --- a/polling/repositories.py +++ b/polling/repositories.py @@ -1,8 +1,9 @@ -from django.shortcuts import get_object_or_404 +import json + +from django_celery_beat.models import IntervalSchedule, PeriodicTask from .models import PollingSchedule -from django_celery_beat.models import PeriodicTask, IntervalSchedule -import json + class PollingRepository: @staticmethod @@ -14,13 +15,14 @@ def get_all_active(poll_type): @staticmethod def get_by_app_id(app_id, poll_type): - return get_object_or_404(PollingSchedule, app_id=app_id, poll_type=poll_type, is_active=True) + return PollingSchedule.objects.filter(app_id=app_id, poll_type=poll_type, is_active=True) @staticmethod def create(app_id, interval_hours, poll_type, periodic_task): - from django.utils.timezone import now from datetime import timedelta + from django.utils.timezone import now + start = now() schedule = PollingSchedule.objects.create( app_id=app_id, @@ -32,21 +34,21 @@ def create(app_id, interval_hours, poll_type, periodic_task): periodic_task=periodic_task, ) return schedule - + @staticmethod def get_or_create_interval_schedule(interval_hours): schedule, _ = IntervalSchedule.objects.get_or_create( - every=interval_hours, - period=IntervalSchedule.HOURS, + every=interval_hours, + period=IntervalSchedule.HOURS, ) return schedule - + @staticmethod def create_periodic_task(schedule, app_id, poll_type, task): name = f"{poll_type}-polling-{app_id}" if PeriodicTask.objects.filter(name=name).exists(): raise ValueError("PeriodicTask already exists with this name") - + task = PeriodicTask.objects.create( interval=schedule, name=name, @@ -54,9 +56,9 @@ def create_periodic_task(schedule, app_id, poll_type, task): args=json.dumps([app_id, poll_type]), ) return task - + @staticmethod def delete(instance): if instance.periodic_task: instance.periodic_task.delete() - instance.delete() \ No newline at end of file + instance.delete() diff --git a/polling/serializers.py b/polling/serializers.py index f4a9313..7dcfb50 100644 --- a/polling/serializers.py +++ b/polling/serializers.py @@ -1,13 +1,15 @@ +from django_celery_beat.models import IntervalSchedule, PeriodicTask from rest_framework import serializers from polling.models import PollingSchedule -from django_celery_beat.models import IntervalSchedule, PeriodicTask + class IntervalScheduleSerializer(serializers.ModelSerializer): class Meta: model = IntervalSchedule fields = ["id", "every", "period"] + class PeriodicTaskSerializer(serializers.ModelSerializer): interval = IntervalScheduleSerializer(read_only=True) @@ -15,6 +17,7 @@ class Meta: model = PeriodicTask fields = ["id", "name", "task", "args", "enabled", "last_run_at", "interval"] + class PollingScheduleSerializer(serializers.ModelSerializer): periodic_task = PeriodicTaskSerializer(read_only=True) diff --git a/polling/services.py b/polling/services.py index 8779934..c5f6022 100644 --- a/polling/services.py +++ b/polling/services.py @@ -1,11 +1,16 @@ -from .repositories import PollingRepository -from source.services import SourceService -from review.repositories import ReviewRepository -from polling.tasks import run_polling_task -from celery import current_app -from datetime import timedelta +from datetime import datetime + from django.utils.timezone import now +from metric.models import Metric, MetricValue +from polling.tasks import run_polling_task +from review.repositories import ReviewRepository +from source.models import Source +from source.services import SourceService + +from .repositories import PollingRepository + + class PollingService: def __init__(self): self.repo = PollingRepository() @@ -16,23 +21,6 @@ def list_active_polling_schedules(self, poll_type=None): def get_polling_schedule(self, app_id, poll_type=None): return self.repo.get_by_app_id(app_id, poll_type) - - def poll_reviews(self, app_id, date_from=None, date_to=None): - adapters = self.source_service.load_sources() - results = [] - - for adapter in adapters: - reviews = adapter.fetch_reviews(app_id=app_id, date_from=date_from, date_to=date_to) - saved_count = ReviewRepository().save_reviews(app_id, reviews) - - results.append({ - "adapter": adapter.name, - "fetched": len(reviews), - "saved": saved_count, - "reviews": reviews, - }) - - return results def create_polling_schedule(self, app_id, interval_hours=None, poll_type=None): existing = self.get_polling_schedule(app_id, poll_type) @@ -40,17 +28,96 @@ def create_polling_schedule(self, app_id, interval_hours=None, poll_type=None): raise ValueError(f"Polling already scheduled for app {app_id} and type {poll_type}") interval_schedule = self.repo.get_or_create_interval_schedule(interval_hours) - periodic_task = self.repo.create_periodic_task(interval_schedule, app_id, poll_type, "polling.tasks.run_polling_task") + periodic_task = self.repo.create_periodic_task( + interval_schedule, app_id, poll_type, "polling.tasks.run_polling_task" + ) schedule = self.repo.create(app_id, interval_hours, poll_type, periodic_task) run_polling_task.delay(app_id, poll_type) return schedule - + def delete_schedule(self, instance): return self.repo.delete(instance) + def poll_reviews(self, app_id, date_from=None, date_to=None): + adapters = self.source_service.load_sources() + results = [] + + for adapter in adapters: + reviews = adapter.fetch_reviews(app_id=app_id, date_from=date_from, date_to=date_to) + saved_count = ReviewRepository().save_reviews(app_id, reviews) + + results.append( + { + "adapter": adapter.name, + "fetched": len(reviews), + "saved": saved_count, + "reviews": reviews, + } + ) + + return results + def poll_metrics(self, app_id): - ... + adapters = self.source_service.load_sources() + print("[🔁] Iniciant recollida de mètriques...") + + metrics = list(Metric.objects.prefetch_related("sources").all()) + + # 🔀 Separa mètriques + external_metrics = [m for m in metrics if not m.is_internal] + internal_metrics = [m for m in metrics if m.is_internal] + + # ⚡ Caches + metric_cache = {m.code: m for m in metrics} + source_cache = {a.code: Source.objects.get(code=a.code) for a in adapters} + + # 🔁 EXTERNAL: Per adapters + for adapter in adapters: + print("adapter actual: ", adapter.code) + source_code = adapter.code + available_metrics = [ + m.code + for m in external_metrics + if source_code in m.sources.values_list("code", flat=True) + ] + if not available_metrics: + continue + + try: + result = adapter.fetch(app_id, available_metrics) + except Exception as e: + print(f"❌ Error en adapter `{source_code}`: {e}") + continue + + for metric_code, value in result.items(): + if value in [None, ""]: + print(f" ⚠️ {metric_code} no ha retornat valor.") + continue + + metric = metric_cache[metric_code] + source = source_cache[source_code] + + already_exists = MetricValue.objects.filter( + app_id=app_id, metric=metric, source=source, retrieved_at__date=now().date() + ).exists() + + if already_exists: + print(f" ℹ️ Ja existia: {metric_code} — {value}") + continue + + MetricValue.objects.create( + app_id=app_id, + metric=metric, + source=source, + value=value, + retrieved_at=datetime.now(), + ) + print(f" ✅ Guardat {metric_code}: {value}") + # 🔁 INTERNAL: opcional (implementa si tens estratègies internes) + if internal_metrics: + print("\n📦 Tractament de mètriques internes no implementat (pendents).") + print("\n✅ Procés de recollida completat.") diff --git a/polling/tasks.py b/polling/tasks.py index 47d03e5..a0d3619 100644 --- a/polling/tasks.py +++ b/polling/tasks.py @@ -1,20 +1,24 @@ -from celery import shared_task import time +from celery import shared_task + + @shared_task def run_polling_task(app_id, poll_type): from polling.services import PollingService + service = PollingService() if poll_type == "metrics": - ##return service.poll_metrics(app_id) print("poll_metrics completat") + return service.poll_metrics(app_id) elif poll_type == "reviews": - ##return service.poll_reviews(app_id print("poll_reviews completat") + return service.poll_reviews(app_id) else: raise ValueError(f"Unknown poll_type: {poll_type}") + @shared_task def prueba_celery(): print("Executant tasca de prova...") diff --git a/polling/urls.py b/polling/urls.py index f62dcd9..0a05ddd 100644 --- a/polling/urls.py +++ b/polling/urls.py @@ -4,11 +4,9 @@ polling_retrieve_view = PollingScheduleViewSet.as_view({"get": "retrieve"}) polling_list_view = PollingScheduleViewSet.as_view({"get": "list"}) -polling_detail_view = PollingScheduleViewSet.as_view({ - "get": "retrieve", - "post": "create", - "delete": "destroy" -}) +polling_detail_view = PollingScheduleViewSet.as_view( + {"get": "retrieve", "post": "create", "delete": "destroy"} +) polling_review_view = PollingScheduleViewSet.as_view({"post": "poll_reviews"}) urlpatterns = [ diff --git a/polling/views.py b/polling/views.py index 94bdfe2..a5b6bba 100644 --- a/polling/views.py +++ b/polling/views.py @@ -5,7 +5,7 @@ from polling.serializers import PollingScheduleSerializer from polling.services import PollingService -from app.services import AppService + class PollingScheduleViewSet(viewsets.ViewSet): service = PollingService() @@ -51,7 +51,7 @@ def create(self, request, id=None, poll_type=None): return Response({"detail": str(e)}, status=409) return Response(PollingScheduleSerializer(schedule).data, status=201) - + @extend_schema( parameters=[OpenApiParameter(name="id", required=True, type=int, location="path")], responses={204: None}, @@ -61,7 +61,7 @@ def destroy(self, request, id=None, poll_type=None): schedule = self.service.get_polling_schedule(id, poll_type) self.service.delete_schedule(schedule) return Response(status=status.HTTP_204_NO_CONTENT) - + @extend_schema( request=None, parameters=[ @@ -102,4 +102,4 @@ def poll_reviews(self, request, id=None): result = self.service.poll_reviews(app_id=id, date_from=date_from, date_to=date_to) - return Response(result) \ No newline at end of file + return Response(result) diff --git a/review/services.py b/review/services.py index ec4e85c..7c23b28 100644 --- a/review/services.py +++ b/review/services.py @@ -1,7 +1,5 @@ -from rest_framework.exceptions import NotFound from django.core.exceptions import ObjectDoesNotExist - -from source.adapters.google_play_scraper import GooglePlayScraperAdapter +from rest_framework.exceptions import NotFound from .repositories import ReviewRepository @@ -18,5 +16,3 @@ def get_review(self, review_id): return self.repo.get_by_id(review_id) except ObjectDoesNotExist: raise NotFound(f"The review with ID '{review_id}' is not registered.") - - diff --git a/review/views.py b/review/views.py index 505ddbf..006e335 100644 --- a/review/views.py +++ b/review/views.py @@ -1,4 +1,4 @@ -from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_schema +from drf_spectacular.utils import OpenApiParameter, extend_schema from rest_framework import viewsets from rest_framework.response import Response diff --git a/source/adapters/base.py b/source/adapters/base.py index 8f23139..524dbba 100644 --- a/source/adapters/base.py +++ b/source/adapters/base.py @@ -23,4 +23,4 @@ def supports_metric(self, metric: str) -> bool: ... def fetch(self, app_id: int, metrics: list[str]) -> dict[str, str]: ... def fetch_reviews(self, app_id: int, date_from=None, date_to=None): - return [] \ No newline at end of file + return [] diff --git a/source/adapters/google_play_scraper.py b/source/adapters/google_play_scraper.py index 7797638..981d37c 100644 --- a/source/adapters/google_play_scraper.py +++ b/source/adapters/google_play_scraper.py @@ -80,7 +80,7 @@ def fetch_reviews(self, app_id: int, date_from=None, date_to=None): app.playstore_id, lang="en", country="us", - count=10, + count=count_per_request, continuation_token=next_token, ) count += 1 diff --git a/source/adapters/reddit.py b/source/adapters/reddit.py index 4eb1a77..52613dd 100644 --- a/source/adapters/reddit.py +++ b/source/adapters/reddit.py @@ -1,7 +1,6 @@ import os import praw - from rest_framework.exceptions import APIException from app.services import AppService diff --git a/source/services.py b/source/services.py index 4ccdf38..daad8cb 100644 --- a/source/services.py +++ b/source/services.py @@ -1,6 +1,7 @@ -from source.adapters.base import SourceAdapter -from rest_framework.exceptions import NotFound from django.core.exceptions import ObjectDoesNotExist +from rest_framework.exceptions import NotFound + +from source.adapters.base import SourceAdapter from .repositories import SourceRepository From dc98d41d71236489cd4942815e6bc737d526a881 Mon Sep 17 00:00:00 2001 From: Anyer Date: Mon, 5 May 2025 23:43:24 +0200 Subject: [PATCH 40/42] =?UTF-8?q?feature(poll=5Fmetrics):=20Implementar=20?= =?UTF-8?q?funci=C3=B3=20per=20a=20la=20recollida=20peri=C3=B2dica=20de=20?= =?UTF-8?q?m=C3=A8triques?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- polling/repositories.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polling/repositories.py b/polling/repositories.py index 429ac1d..6aa73e1 100644 --- a/polling/repositories.py +++ b/polling/repositories.py @@ -15,7 +15,7 @@ def get_all_active(poll_type): @staticmethod def get_by_app_id(app_id, poll_type): - return PollingSchedule.objects.filter(app_id=app_id, poll_type=poll_type, is_active=True) + return PollingSchedule.objects.get(app_id=app_id, poll_type=poll_type) @staticmethod def create(app_id, interval_hours, poll_type, periodic_task): From 5d3e53acb9947b7f8b3cb7b0907126cad4fc643a Mon Sep 17 00:00:00 2001 From: Anyer Date: Sun, 11 May 2025 13:41:22 +0200 Subject: [PATCH 41/42] =?UTF-8?q?feature(re-miner):=20Configurar=20la=20co?= =?UTF-8?q?nnexi=C3=B3=20als=20microserveis=20de=20RE-Miner=202.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refs #60 --- config/settings.py | 10 +++- docker-compose.yml | 20 +++++++ polling/repositories.py | 9 +++- polling/serializers.py | 11 +--- polling/services.py | 70 ++++++++++++++++++++++++- requirements.txt | Bin 1152 -> 2488 bytes review/models.py | 7 ++- source/adapters/google_play_scraper.py | 2 +- 8 files changed, 114 insertions(+), 15 deletions(-) diff --git a/config/settings.py b/config/settings.py index 9c04fe3..2a83d36 100644 --- a/config/settings.py +++ b/config/settings.py @@ -52,18 +52,26 @@ "polling", "drf_spectacular", "django_celery_beat", + "corsheaders", ] MIDDLEWARE = [ + "corsheaders.middleware.CorsMiddleware", + "django.middleware.common.CommonMiddleware", "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", - "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", ] +CORS_ALLOWED_ORIGINS = [ + "https://editor.swagger.io", + "http://localhost:3000", + "https://hn13e-puma-hackers.github.io", +] + ROOT_URLCONF = "config.urls" TEMPLATES = [ diff --git a/docker-compose.yml b/docker-compose.yml index 8cff519..bbd7dc4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,3 +35,23 @@ services: - .:/app depends_on: - redis + + re-miner-polarity: + build: + context: ../RE-Miner-polarity-analysis + ports: + - "3010:3010" + command: uvicorn main:app --host 0.0.0.0 --port 3010 + volumes: + - ../RE-Miner-polarity-analysis:/app + working_dir: /app + + re-miner-type: + build: + context: ../RE-Miner-type-analysis + ports: + - "3011:3011" + command: uvicorn main:app --host 0.0.0.0 --port 3011 + volumes: + - ../RE-Miner-type-analysis:/app + working_dir: /app \ No newline at end of file diff --git a/polling/repositories.py b/polling/repositories.py index 6aa73e1..e4578a9 100644 --- a/polling/repositories.py +++ b/polling/repositories.py @@ -15,7 +15,10 @@ def get_all_active(poll_type): @staticmethod def get_by_app_id(app_id, poll_type): - return PollingSchedule.objects.get(app_id=app_id, poll_type=poll_type) + try: + return PollingSchedule.objects.get(app_id=app_id, poll_type=poll_type) + except PollingSchedule.DoesNotExist: + return None @staticmethod def create(app_id, interval_hours, poll_type, periodic_task): @@ -62,3 +65,7 @@ def delete(instance): if instance.periodic_task: instance.periodic_task.delete() instance.delete() + + @staticmethod + def exists(app_id, poll_type): + return PollingSchedule.objects.filter(app_id=app_id, poll_type=poll_type).exists() diff --git a/polling/serializers.py b/polling/serializers.py index 7dcfb50..5f53b8a 100644 --- a/polling/serializers.py +++ b/polling/serializers.py @@ -1,21 +1,14 @@ -from django_celery_beat.models import IntervalSchedule, PeriodicTask +from django_celery_beat.models import PeriodicTask from rest_framework import serializers from polling.models import PollingSchedule -class IntervalScheduleSerializer(serializers.ModelSerializer): - class Meta: - model = IntervalSchedule - fields = ["id", "every", "period"] - - class PeriodicTaskSerializer(serializers.ModelSerializer): - interval = IntervalScheduleSerializer(read_only=True) class Meta: model = PeriodicTask - fields = ["id", "name", "task", "args", "enabled", "last_run_at", "interval"] + fields = ["id", "name", "task", "args", "enabled", "last_run_at"] class PollingScheduleSerializer(serializers.ModelSerializer): diff --git a/polling/services.py b/polling/services.py index c5f6022..51e320c 100644 --- a/polling/services.py +++ b/polling/services.py @@ -1,5 +1,6 @@ from datetime import datetime +import requests from django.utils.timezone import now from metric.models import Metric, MetricValue @@ -16,6 +17,42 @@ def __init__(self): self.repo = PollingRepository() self.source_service = SourceService() + def _analyze_reviews(self, reviews, service_url: str, label: str = "RE-Miner"): + """ + Envia una llista de ressenyes a un servei RE-Miner i retorna la resposta. + + reviews: List[Dict] amb clau 'text' i opcionalment 'reviewId'. + service_url: URL del servei RE-Miner. + label: Nom del servei per mostrar en errors. + """ + try: + payload = { + "reviews": [ + {"reviewId": str(r.get("reviewId") or idx), "text": r["text"]} + for idx, r in enumerate(reviews) + ] + } + response = requests.post(service_url, json=payload, timeout=10) + response.raise_for_status() + return response.json() + except Exception as e: + print(f"[{label} ❌] Error analitzant ressenyes: {e}") + return [] + + def _analyze_review_polarity(self, reviews): + return self._analyze_reviews( + reviews, + service_url="http://re-miner-polarity:3010/analyze-polarity?polarity-service=MLP", + label="RE-Miner (Polarity)", + ) + + def _analyze_review_type(self, reviews): + return self._analyze_reviews( + reviews, + service_url="http://re-miner-type:3011/analyze-type?type-service=DISTILBERT", + label="RE-Miner (Type)", + ) + def list_active_polling_schedules(self, poll_type=None): return self.repo.get_all_active(poll_type) @@ -23,8 +60,7 @@ def get_polling_schedule(self, app_id, poll_type=None): return self.repo.get_by_app_id(app_id, poll_type) def create_polling_schedule(self, app_id, interval_hours=None, poll_type=None): - existing = self.get_polling_schedule(app_id, poll_type) - if existing: + if self.repo.exists(app_id, poll_type): raise ValueError(f"Polling already scheduled for app {app_id} and type {poll_type}") interval_schedule = self.repo.get_or_create_interval_schedule(interval_hours) @@ -46,6 +82,36 @@ def poll_reviews(self, app_id, date_from=None, date_to=None): for adapter in adapters: reviews = adapter.fetch_reviews(app_id=app_id, date_from=date_from, date_to=date_to) + if not reviews: + results.append( + { + "adapter": adapter.name, + "fetched": 0, + "saved": 0, + "reviews": [], + } + ) + continue + + # 🧪 Preparem les dades per RE-Miner (agafa 'content' com a 'text') + for idx, r in enumerate(reviews): + r["reviewId"] = r.get("reviewId", str(idx)) # assegura que sempre hi ha un id + + prepared_reviews = [{"reviewId": r["reviewId"], "text": r["content"]} for r in reviews] + polarity_result = self._analyze_review_polarity(prepared_reviews) + type_result = self._analyze_review_type(prepared_reviews) + + # 🔁 Mapegem la polaritat a cada review original + polarity_by_id = { + r["reviewId"]: r["polarity"] for r in polarity_result.get("reviews", []) + } + + type_by_id = {r["reviewId"]: r["type"] for r in type_result.get("reviews", [])} + + for review in reviews: + review["polarity"] = polarity_by_id.get(review["reviewId"]) + review["type"] = type_by_id.get(review["reviewId"]) + saved_count = ReviewRepository().save_reviews(app_id, reviews) results.append( diff --git a/requirements.txt b/requirements.txt index e5defd0127f8a089efeb154d415a36f713a748ce..2b4228eff179dd60a48e6f50bd7451ca75243069 100644 GIT binary patch literal 2488 zcmZ{mO>fg+5QN_|5A-YRxB#4lu>mu2}eil55r_@%1e>Dia3u!D_f7TK}xRlOCmwvRuP4G(gh z_dP6SUzdmQ06hpY|4kt4l<80=Wn1o*aiibB{2ck=B%{p9GOUT&T)+u@Rog`UwtAkF z^QfGSvTg&zSzt?s#}PYaJ_g_0o>BVn^KW)0fRIAQ(qHOST zh$vJ6o9<#Y%1IQ+Iw8J^2ymzBzGxVS;IJvD zsBe|h_f$+)r1$b|#HTgJG(N<`%vQ>At-77~ZPdp>8JC(ex;NMO&qket5Y;ooRGDj5 zc3}EmO2KauI+#moRAH9$s8uHkvz`Tt1gW>iTZNTwJWD*b%8vhzlhgQ|A zn!Ry!i&rZQSm|RuOcp(OvS^8Edg|ser*IoK)W3{Lr+%67EcVbjUh=CP`ywpWMIz1n zl=p#rp8F4B*+_@SLAi8u9?IQ_(_Qe|lt<-)xvuZs8QoQO#{Gcbss-$5!L@cm#6;)) zjwd_r4*u{ssPj}yZueB(*DPoUb;9CC6}ivGLngH1Y3DsztB2fbx-*BnP|7JR>_NTi zY4O%I!wkt@W1DgnlimnJD?C(iCu1+z<+5DqL9=)2KfOVU`Y#54p5*1-BO8eD@fBhc zY2|X&tCE$c%2Qdw`$iSHb8H3wWBYu;$P1_0&)9nv{5H}A`tgm$mq~(mr(E2Jmt+6F z21b;1zNv@r_~*I7tlC(H%YNca+wI0=+Xv>&mGV{QQ-&5E?R> zr<~bCbmGYTw(v}_aL10~f=irl0Fc84|Jw5;yeE0Dcitf0EDZzzg79;1Az9^f5uSUu?aV!BN8-&kl zs&XV-j9s0TK_v!@iC%1276vqNt3<*Q%S#Et-6aZa`i#P@KqfI< z2P+(>>d;Rr8*vt*H#kS^Qw)649Sx7@XbP=`VY(K;Y8^u^nD&+Y5?dTvt6$siZ$Ez= zT(>F_jCUMr941QS?M?_xegP$k-d5};MC%TpN^#6p1b&!AKxY2f&3~2uHy7niNfOUh zl`W#z%l|q64K`Qh6DvdCUAbpgU52)x5Z@jpdrK2c=dR`aICB@ll}n5AYttQ diff --git a/review/models.py b/review/models.py index 3616b07..6fef8be 100644 --- a/review/models.py +++ b/review/models.py @@ -12,8 +12,9 @@ class ReviewPolarity(models.TextChoices): class ReviewType(models.TextChoices): BUG = "bug", "Bug" + RATING = "rating", "Rating" FEATURE = "feature", "Feature" - GENERAL = "general", "General" + USER_EXPERIENCE = "user_experience", "User Experience" class ReviewTopic(models.TextChoices): @@ -23,7 +24,11 @@ class ReviewTopic(models.TextChoices): EFFECTIVENESS = "effectiveness", "Effectiveness" EFFICIENCY = "efficiency", "Efficiency" ENJOYABILITY = "enjoyability", "Enjoyability" + GENERAL = "general", "General" LEARNABILITY = "learnability", "Learnability" + N_A = "n/a", "N/A" + RELIABILITY = "reliability", "Reliability" + SAFETY = "safety", "Safety" SECURITY = "security", "Security" USABILITY = "usability", "Usability" diff --git a/source/adapters/google_play_scraper.py b/source/adapters/google_play_scraper.py index 981d37c..ce0ba85 100644 --- a/source/adapters/google_play_scraper.py +++ b/source/adapters/google_play_scraper.py @@ -69,7 +69,7 @@ def fetch_reviews(self, app_id: int, date_from=None, date_to=None): if days_range < 0: raise ValidationError("date_from must be before or equal to date_to.") - count_per_request = min(180 * days_range, 4500) + count_per_request = min(max(1, 180 * days_range), 4500) all_reviews = [] next_token = None From 0e02e5c222a36ab7ead84f67a36d3f6b8928ac51 Mon Sep 17 00:00:00 2001 From: Anyer Date: Sun, 11 May 2025 20:15:04 +0200 Subject: [PATCH 42/42] hotfix(tests): Arreglar tests d'integritat i unitaris. Refs # --- metric/management/__init__.py | 0 metric/management/commands/__init__.py | 0 metric/management/commands/poll_metrics.py | 109 --------------------- metric/management/commands/poll_reviews.py | 17 ---- tests/integration/test_end_to_end.py | 38 ++----- tests/unit/test_generic_metric_strategy.py | 48 --------- 6 files changed, 7 insertions(+), 205 deletions(-) delete mode 100644 metric/management/__init__.py delete mode 100644 metric/management/commands/__init__.py delete mode 100644 metric/management/commands/poll_metrics.py delete mode 100644 metric/management/commands/poll_reviews.py delete mode 100644 tests/unit/test_generic_metric_strategy.py diff --git a/metric/management/__init__.py b/metric/management/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/metric/management/commands/__init__.py b/metric/management/commands/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/metric/management/commands/poll_metrics.py b/metric/management/commands/poll_metrics.py deleted file mode 100644 index 5391166..0000000 --- a/metric/management/commands/poll_metrics.py +++ /dev/null @@ -1,109 +0,0 @@ -from datetime import datetime - -from django.core.management.base import BaseCommand -from django.utils.timezone import now - -from app.models import App -from metric.constants import MetricCodes -from metric.models import Metric, MetricValue -from source.models import Source -from source.services import SourceService - - -def validate_metric_codes(): - # ✅ Validació entre MetricCodes i Metric a BD - codes_in_db = set(Metric.objects.values_list("code", flat=True)) - codes_in_code = { - v for k, v in MetricCodes.__dict__.items() if not k.startswith("__") and not callable(v) - } - - extra_in_db = codes_in_db - codes_in_code - missing_in_db = codes_in_code - codes_in_db - - if extra_in_db: - print("❌ Les següents mètriques estan a la BD però no a MetricCodes:") - for code in sorted(extra_in_db): - print(f" - {code}") - - if missing_in_db: - print("❌ Les següents MetricCodes no estan registrades a la BD:") - for code in sorted(missing_in_db): - print(f" - {code}") - - if not extra_in_db and not missing_in_db: - print("✅ Coherència correcta entre MetricCodes i BD.") - return True - else: - print("❌ Hi ha inconsistències entre MetricCodes i BD.") - return False - - -class Command(BaseCommand): - help = "Recull mètriques per a totes les apps i fonts disponibles" - - def handle(self, *args, **options): - print("[🔁] Iniciant recollida de mètriques...") - - # 🔃 Carrega dades necessàries - adapters = SourceService().load_sources() - apps = App.objects.all() - metrics = list(Metric.objects.prefetch_related("sources").all()) - - # 🔀 Separa mètriques - external_metrics = [m for m in metrics if not m.is_internal] - internal_metrics = [m for m in metrics if m.is_internal] - - # ⚡ Caches - metric_cache = {m.code: m for m in metrics} - source_cache = {a.code: Source.objects.get(code=a.code) for a in adapters} - - # 🔁 EXTERNAL: Per adapters - for app in apps: - print(f"\n🔍 App: {app.code}") - for adapter in adapters: - print("adapter actual: ", adapter.code) - source_code = adapter.code - available_metrics = [ - m.code - for m in external_metrics - if source_code in m.sources.values_list("code", flat=True) - ] - if not available_metrics: - continue - - try: - result = adapter.fetch(app.id, available_metrics) - except Exception as e: - print(f"❌ Error en adapter `{source_code}`: {e}") - continue - - for metric_code, value in result.items(): - if value in [None, ""]: - print(f" ⚠️ {metric_code} no ha retornat valor.") - continue - - metric = metric_cache[metric_code] - source = source_cache[source_code] - - already_exists = MetricValue.objects.filter( - app=app, metric=metric, source=source, retrieved_at__date=now().date() - ).exists() - - if already_exists: - print(f" ℹ️ Ja existia: {metric_code} — {value}") - continue - - MetricValue.objects.create( - app=app, - metric=metric, - source=source, - value=value, - retrieved_at=datetime.now(), - ) - print(f" ✅ Guardat {metric_code}: {value}") - - # 🔁 INTERNAL: opcional (implementa si tens estratègies internes) - if internal_metrics: - print("\n📦 Tractament de mètriques internes no implementat (pendents).") - - print("\n✅ Procés de recollida completat.") diff --git a/metric/management/commands/poll_reviews.py b/metric/management/commands/poll_reviews.py deleted file mode 100644 index 9e244c2..0000000 --- a/metric/management/commands/poll_reviews.py +++ /dev/null @@ -1,17 +0,0 @@ -from django.core.management.base import BaseCommand - -from app.models import App -from source.adapters.google_play_scraper import GooglePlayScraperAdapter - - -class Command(BaseCommand): - help = "Recull las resseynes del dia d'ahir de google play" - - def handle(self, *args, **options): - print("[🔁] Iniciant recollida de ressenyes diàries...") - - apps = App.objects.all() - for app in apps: - GooglePlayScraperAdapter().fetch_reviews(app.id) - - print("✅ Procés completat.") diff --git a/tests/integration/test_end_to_end.py b/tests/integration/test_end_to_end.py index c25906e..9ed79df 100644 --- a/tests/integration/test_end_to_end.py +++ b/tests/integration/test_end_to_end.py @@ -1,45 +1,21 @@ import pytest -from app.models import App -from metric.models import Metric -from metric.strategies.generic import GenericMetricStrategy -from source.constants.source_type import SourceType -from source.models import Source from source.services import SourceService @pytest.mark.django_db -def test_full_flow_with_real_adapter(): - # Setup: crear App, Metric y Source reals - app = App.objects.create( - code="discord", - name="Discord", - description="App de comunicació global", - appstore_id="985746746", - ) - metric = Metric.objects.create(code="average_rating", name="Average Rating", value_type="float") - source = Source.objects.create( - code="itunes", - name="iTunes Search API", - type=SourceType.API, - url="https://itunes.apple.com", - ) - +def test_full_flow_with_real_adapter(dummy_app): # Adapter real (ja ha de formar part del projecte i registrar-se automàticament) adapters = SourceService().load_sources() itunes_adapter = next((a for a in adapters if a.code == "itunes"), None) assert itunes_adapter is not None, "iTunes adapter not found in loaded sources" - strategy = GenericMetricStrategy("average_rating") - values = strategy.compute_all(app.id, [itunes_adapter]) + values = itunes_adapter.fetch(dummy_app.id, ["average_rating"]) - for v in values: - print(f"metric={v.metric.code}, app={v.app.code}, source={v.source.code}, value={v.value}") + # Comprovar que el valor retornat és un diccionari i conté la clau "average_rating" + assert isinstance(values, dict), "Expected values to be a dictionary" + assert "average_rating" in values, "'average_rating' key not found in values" - assert isinstance(values, list) - assert any( - v.metric == metric and v.app == app and v.source == source - for v in values - if v.value is not None - ) + # Comprovar que el valor no és None + assert values["average_rating"] is not None, "'average_rating' value is None" diff --git a/tests/unit/test_generic_metric_strategy.py b/tests/unit/test_generic_metric_strategy.py deleted file mode 100644 index 03224c2..0000000 --- a/tests/unit/test_generic_metric_strategy.py +++ /dev/null @@ -1,48 +0,0 @@ -from datetime import datetime - -import pytest - -from app.models import App -from metric.models import Metric -from metric.strategies.generic import GenericMetricStrategy -from source.constants.source_type import SourceType -from source.models import Source - - -@pytest.mark.django_db -def test_generic_metric_strategy_computes_values(): - # 1. Crear els objectes necessaris a la BD - app = App.objects.create(name="Test App") - metric = Metric.objects.create(code="average_rating", name="Average Rating", value_type="float") - source = Source.objects.create( - code="fake", name="Fake Source", type=SourceType.API, url="http://fake" - ) - - # 2. Adapter fals - class FakeAdapter: - code = "fake" - name = "Fake Source" - type = SourceType.API - url = "http://fake" - supported_metrics = ["average_rating"] - - def supports_metric(self, metric: str) -> bool: - return True - - def fetch(self, app_id: str, metric: str): - return 4.8 - - adapter = FakeAdapter() - - # 3. Executar l'estratègia - strategy = GenericMetricStrategy("average_rating") - result = strategy.compute_all(app.id, [adapter]) - - # 4. Validació - assert len(result) == 1 - value = result[0] - assert value.app == app - assert value.metric == metric - assert value.source == source - assert value.value == 4.8 - assert isinstance(value.retrieved_at, datetime)