From 9a54a8d2e40eeb0270cde9c6c011f54389149830 Mon Sep 17 00:00:00 2001 From: MiqSA Date: Fri, 28 Oct 2022 06:44:30 -0300 Subject: [PATCH 01/13] Add project configurations --- config/__init__.py | 5 ++ config/asgi.py | 16 +++++ config/settings.py | 153 +++++++++++++++++++++++++++++++++++++++++++++ config/urls.py | 23 +++++++ config/wsgi.py | 16 +++++ 5 files changed, 213 insertions(+) create mode 100644 config/__init__.py create mode 100644 config/asgi.py create mode 100644 config/settings.py create mode 100644 config/urls.py create mode 100644 config/wsgi.py diff --git a/config/__init__.py b/config/__init__.py new file mode 100644 index 0000000000..8319f0e447 --- /dev/null +++ b/config/__init__.py @@ -0,0 +1,5 @@ +""" +TODO: Create env. +TODO: Registre the apps + +""" \ No newline at end of file diff --git a/config/asgi.py b/config/asgi.py new file mode 100644 index 0000000000..9502b7fd4b --- /dev/null +++ b/config/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for config project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.1/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') + +application = get_asgi_application() diff --git a/config/settings.py b/config/settings.py new file mode 100644 index 0000000000..52c46e681e --- /dev/null +++ b/config/settings.py @@ -0,0 +1,153 @@ +from pathlib import Path +import os +from dotenv import load_dotenv + +load_dotenv() + +BASE_DIR = Path(__file__).resolve().parent.parent + +SECRET_KEY = os.getenv('SECRET_KEY', 'MY_SECRET_KEY') +DEVELOP_LOCAL = True + +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django.contrib.sites', + 'rest_framework', + 'rest_framework.authtoken', + 'dj_rest_auth', + 'allauth', + 'allauth.account', + 'dj_rest_auth.registration', + 'allauth.socialaccount', + 'drf_yasg', + 'corsheaders', + 'apps.dataexplore', + 'apps.users', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'config.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'config.wsgi.application' + + + + +if DEVELOP_LOCAL == True: + DATABASES = { + 'default': { + 'ENGINE': os.getenv('Engine'), + 'NAME': os.getenv('Name'), + 'USER': os.getenv('User'), + 'PASSWORD': os.getenv('Password'), + 'HOST': os.getenv('Host'), + 'PORT': os.getenv('Port'), + } + } +else: + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': os.environ.get('POSTGRES_NAME'), + 'USER': os.environ.get('POSTGRES_USER'), + 'PASSWORD': os.environ.get('POSTGRES_PASSWORD'), + 'HOST': 'db', + 'PORT': 5432, + } + } + +PASSWORD_HASHERS = [ + 'django.contrib.auth.hashers.Argon2PasswordHasher', + 'django.contrib.auth.hashers.PBKDF2PasswordHasher', + 'django.contrib.auth.hashers.PBKDF2SHA1PasswordHasher', + 'django.contrib.auth.hashers.BCryptSHA256PasswordHasher', + 'django.contrib.auth.hashers.ScryptPasswordHasher', +] + + + + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_TZ = True + + + +STATIC_URL = 'static/' + + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + + +EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' + + +REST_FRAMEWORK = { + 'DEFAULT_AUTHENTICATION_CLASSES': ( + 'rest_framework.authentication.SessionAuthentication', + 'rest_framework.authentication.TokenAuthentication', + 'dj_rest_auth.jwt_auth.JWTCookieAuthentication' + ), + 'DEFAULT_SCHEMA_CLASS': 'rest_framework.schemas.coreapi.AutoSchema' +} + + +CORS_ORIGIN_ALLOW_ALL = True diff --git a/config/urls.py b/config/urls.py new file mode 100644 index 0000000000..6ddae56352 --- /dev/null +++ b/config/urls.py @@ -0,0 +1,23 @@ +from django.contrib import admin +from django.urls import path, include, re_path +from rest_framework import permissions +from drf_yasg.views import get_schema_view +from drf_yasg import openapi + +schema_view = get_schema_view( + openapi.Info( + title='API', + default_version='v1', + ), + public=False, + permission_classes=[permissions.AllowAny], +) + +urlpatterns = [ + path('admin/', admin.site.urls, name='admin'), + path('users/', include('apps.users.urls'), name='users'), + path('dataexplore/', include('apps.dataexplore.urls'), name='dataexplore'), + path('accounts/', include('dj_rest_auth.urls'), name='account'), + re_path(r'^swagger/$', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'), + re_path(r'^redoc/$', schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc'), + ] diff --git a/config/wsgi.py b/config/wsgi.py new file mode 100644 index 0000000000..3d2dc4565e --- /dev/null +++ b/config/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for config project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.1/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') + +application = get_wsgi_application() From 90bcdffb84b0ba932fbc5f04d14bea219165bb73 Mon Sep 17 00:00:00 2001 From: MiqSA Date: Fri, 28 Oct 2022 09:31:46 -0300 Subject: [PATCH 02/13] Add apps users and dataexplore --- apps/dataexplore/__init__.py | 0 apps/dataexplore/admin.py | 3 ++ apps/dataexplore/apps.py | 6 ++++ apps/dataexplore/forms.py | 10 ++++++ apps/dataexplore/models.py | 17 ++++++++++ apps/dataexplore/tests.py | 3 ++ apps/dataexplore/urls.py | 11 +++++++ apps/dataexplore/views.py | 32 ++++++++++++++++++ apps/users/__init__.py | 4 +++ apps/users/admin.py | 3 ++ apps/users/apps.py | 6 ++++ apps/users/models.py | 4 +++ apps/users/serializers.py | 14 ++++++++ apps/users/tests.py | 3 ++ apps/users/urls.py | 12 +++++++ apps/users/views.py | 64 ++++++++++++++++++++++++++++++++++++ 16 files changed, 192 insertions(+) create mode 100644 apps/dataexplore/__init__.py create mode 100644 apps/dataexplore/admin.py create mode 100644 apps/dataexplore/apps.py create mode 100644 apps/dataexplore/forms.py create mode 100644 apps/dataexplore/models.py create mode 100644 apps/dataexplore/tests.py create mode 100644 apps/dataexplore/urls.py create mode 100644 apps/dataexplore/views.py create mode 100644 apps/users/__init__.py create mode 100644 apps/users/admin.py create mode 100644 apps/users/apps.py create mode 100644 apps/users/models.py create mode 100644 apps/users/serializers.py create mode 100644 apps/users/tests.py create mode 100644 apps/users/urls.py create mode 100644 apps/users/views.py diff --git a/apps/dataexplore/__init__.py b/apps/dataexplore/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/dataexplore/admin.py b/apps/dataexplore/admin.py new file mode 100644 index 0000000000..8c38f3f3da --- /dev/null +++ b/apps/dataexplore/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/apps/dataexplore/apps.py b/apps/dataexplore/apps.py new file mode 100644 index 0000000000..d16ce54853 --- /dev/null +++ b/apps/dataexplore/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class DataexploreConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'apps.dataexplore' diff --git a/apps/dataexplore/forms.py b/apps/dataexplore/forms.py new file mode 100644 index 0000000000..0bc5dc1fe5 --- /dev/null +++ b/apps/dataexplore/forms.py @@ -0,0 +1,10 @@ +from django import forms +from .models import DataModel + + +class DataForm(forms.ModelForm): + class Meta: + model = DataModel + fields = [ + "filename", + ] \ No newline at end of file diff --git a/apps/dataexplore/models.py b/apps/dataexplore/models.py new file mode 100644 index 0000000000..fe4ecd3d20 --- /dev/null +++ b/apps/dataexplore/models.py @@ -0,0 +1,17 @@ +from django.db import models +from datetime import datetime + +class DataModel(models.Model): + """This class define the data model.""" + filename = models.CharField(max_length=255) + date_created = models.DateTimeField(default=datetime.now) + last_update = models.DateTimeField(default=datetime.now, blank=True) + + def __str__(self) -> str: + return str(self.filename) + +class TransactionModel(models.Model): + pass + +class TransactionTypeModel(models.Model): + pass diff --git a/apps/dataexplore/tests.py b/apps/dataexplore/tests.py new file mode 100644 index 0000000000..7ce503c2dd --- /dev/null +++ b/apps/dataexplore/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/apps/dataexplore/urls.py b/apps/dataexplore/urls.py new file mode 100644 index 0000000000..245289fb94 --- /dev/null +++ b/apps/dataexplore/urls.py @@ -0,0 +1,11 @@ +"""Here the urls app data-explore are registered.""" + +from django.urls import path +from apps.dataexplore import views # pylint: disable=import-error + + +urlpatterns = [ + path('upload-files', views.upload_files, name='upload-files'), + path('transform', views.data_transform, name='transform-data'), + path('dashboard', views.dashboard, name='dashboard'), +] \ No newline at end of file diff --git a/apps/dataexplore/views.py b/apps/dataexplore/views.py new file mode 100644 index 0000000000..fdd84d8d1f --- /dev/null +++ b/apps/dataexplore/views.py @@ -0,0 +1,32 @@ +from django.shortcuts import render +from drf_yasg.utils import swagger_auto_schema +from rest_framework.decorators import api_view + +from .forms import DataForm + +# Create your views here. +""" +TODO: Create the view to handle with divisions informations +""" + +@api_view(['POST']) +def upload_files(request): # pylint: disable=unused-argument + """This view make the upload files.""" + context = {} + form = DataForm(request.POST or None) + if form.is_valid(): + form.save() + context['form'] = form + return render(request, "create.html", context) + +@api_view(['GET']) +def data_transform(): + """This view transform the data from file in informations to database.""" + pass + +@api_view(['GET']) +def dashboard(): + """This views brings informations about the transactions.""" + pass + + diff --git a/apps/users/__init__.py b/apps/users/__init__.py new file mode 100644 index 0000000000..26105422ed --- /dev/null +++ b/apps/users/__init__.py @@ -0,0 +1,4 @@ +""" +TODO: Create Oath +TODO: Create CRUD +""" \ No newline at end of file diff --git a/apps/users/admin.py b/apps/users/admin.py new file mode 100644 index 0000000000..8c38f3f3da --- /dev/null +++ b/apps/users/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/apps/users/apps.py b/apps/users/apps.py new file mode 100644 index 0000000000..2bb189ca68 --- /dev/null +++ b/apps/users/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class UsersConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'apps.users' diff --git a/apps/users/models.py b/apps/users/models.py new file mode 100644 index 0000000000..b096caa1f6 --- /dev/null +++ b/apps/users/models.py @@ -0,0 +1,4 @@ +from django.db import models + +# Create your models here. + diff --git a/apps/users/serializers.py b/apps/users/serializers.py new file mode 100644 index 0000000000..54a9698200 --- /dev/null +++ b/apps/users/serializers.py @@ -0,0 +1,14 @@ +from django.contrib.auth.models import User, Group +from rest_framework import serializers + + +class UserSerializer(serializers.ModelSerializer): + class Meta: + model = User + fields = ['username', 'email'] + + +class GroupSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = Group + fields = ['name'] \ No newline at end of file diff --git a/apps/users/tests.py b/apps/users/tests.py new file mode 100644 index 0000000000..7ce503c2dd --- /dev/null +++ b/apps/users/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/apps/users/urls.py b/apps/users/urls.py new file mode 100644 index 0000000000..fb7da84072 --- /dev/null +++ b/apps/users/urls.py @@ -0,0 +1,12 @@ +"""Here the urls app users are registered.""" + +from django.urls import path +from apps.users import views # pylint: disable=import-error + + +urlpatterns = [ + path('create', views.create_user, name='user_create'), + path('list', views.list_users, name='user_list'), + path('update/', views.update_user, name='user_update'), + path('delete/', views.delete_user, name='user_delete'), +] diff --git a/apps/users/views.py b/apps/users/views.py new file mode 100644 index 0000000000..8a4f670710 --- /dev/null +++ b/apps/users/views.py @@ -0,0 +1,64 @@ +"""In this module the user views are registered.""" +from django.shortcuts import get_object_or_404, render +from django.contrib.auth.models import User +from rest_framework.decorators import api_view +from rest_framework.response import Response +from rest_framework import status, serializers + +from apps.users.serializers import UserSerializer +# from .forms import UsersForm + + +@api_view(['POST']) +def create_user(request): + """This view create user.""" + user = UserSerializer(data=request.data) + if User.objects.filter(**request.data).exists(): + raise serializers.ValidationError('This data already exists') + + if user.is_valid(): + User.objects.create_user(**request.data) + return Response(user.data) + return Response(status=status.HTTP_404_NOT_FOUND) + + +@api_view(['GET']) +def list_users(request): + """This view list users.""" + if request.query_params: + users = User.objects.get(**request.query_param.dict()) + else: + users = User.objects.all() + if users: + data_users = UserSerializer(users, many=True) + return Response(data_users.data) + return Response(status.HTTP_404_NOT_FOUND) + + +@api_view(['POST']) +def update_user(request, user_id): + """This view update user.""" + user = User.objects.get(pk=user_id) + data = UserSerializer(instance=user, data=request.data) + if data.is_valid(): + data.save() + return Response(data.data) + return Response(status=status.HTTP_404_NOT_FOUND) + + +@api_view(['DELETE']) +def delete_user(request, user_id): # pylint: disable=unused-argument + """This view delete user.""" + user = get_object_or_404(User, pk=user_id) + user.delete() + return Response(status=status.HTTP_202_ACCEPTED) + + +# def create_view(request): # pylint: disable=unused-argument +# """This view create user by UsersForm.""" +# context = {} +# form = UsersForm(request.POST or None) +# if form.is_valid(): +# form.save() +# context['form'] = form +# return render(request, "create.html", context) From 006fa7cc49d02e4c989545bfdb157e77eba24aab Mon Sep 17 00:00:00 2001 From: MiqSA Date: Fri, 28 Oct 2022 09:40:47 -0300 Subject: [PATCH 03/13] Add requirements.txt --- requirements.txt | 55 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..f3656ce78b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,55 @@ +asgiref==3.5.2 +astroid==2.12.12 +attrs==22.1.0 +certifi==2022.9.24 +cffi==1.15.1 +charset-normalizer==2.1.1 +coreapi==2.3.3 +coreschema==0.0.4 +cryptography==38.0.1 +defusedxml==0.7.1 +dill==0.3.6 +dj-rest-auth==2.2.5 +Django==4.1.2 +django-allauth==0.51.0 +django-cors-headers==3.13.0 +djangorestframework==3.14.0 +djangorestframework-simplejwt==5.2.2 +drf-yasg==1.21.4 +flake8==5.0.4 +idna==3.4 +inflection==0.5.1 +iniconfig==1.1.1 +isort==5.10.1 +itypes==1.2.0 +Jinja2==3.1.2 +lazy-object-proxy==1.7.1 +MarkupSafe==2.1.1 +mccabe==0.7.0 +mysqlclient==2.1.1 +oauthlib==3.2.2 +packaging==21.3 +platformdirs==2.5.2 +pluggy==1.0.0 +py==1.11.0 +pycodestyle==2.9.1 +pycparser==2.21 +pydocstringformatter==0.7.2 +pyflakes==2.5.0 +PyJWT==2.6.0 +pylint==2.15.5 +pyparsing==3.0.9 +pytest==7.1.3 +python-dotenv==0.21.0 +python3-openid==3.2.0 +pytz==2022.5 +requests==2.28.1 +requests-oauthlib==1.3.1 +ruamel.yaml==0.17.21 +ruamel.yaml.clib==0.2.7 +sqlparse==0.4.3 +tomli==2.0.1 +tomlkit==0.11.5 +uritemplate==4.1.1 +urllib3==1.26.12 +wrapt==1.14.1 From 53db87f204ba5e146a55fb8fa70d0aea2ece215d Mon Sep 17 00:00:00 2001 From: MiqSA Date: Fri, 28 Oct 2022 09:45:49 -0300 Subject: [PATCH 04/13] Add settings and templates --- config/settings.py | 17 +++++++++++++++-- manage.py | 22 ++++++++++++++++++++++ templates/create.html | 11 +++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) create mode 100755 manage.py create mode 100644 templates/create.html diff --git a/config/settings.py b/config/settings.py index 52c46e681e..fd687ab59b 100644 --- a/config/settings.py +++ b/config/settings.py @@ -23,6 +23,7 @@ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + 'drf_yasg', 'django.contrib.sites', 'rest_framework', 'rest_framework.authtoken', @@ -31,7 +32,6 @@ 'allauth.account', 'dj_rest_auth.registration', 'allauth.socialaccount', - 'drf_yasg', 'corsheaders', 'apps.dataexplore', 'apps.users', @@ -52,7 +52,7 @@ TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], + 'DIRS': [os.path.join(BASE_DIR, 'templates')], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ @@ -151,3 +151,16 @@ CORS_ORIGIN_ALLOW_ALL = True + + +SWAGGER_SETTINGS = { + 'SECURITY_DEFINITIONS': { + 'basic': { + 'type': 'basic' + } + }, +} + +REDOC_SETTINGS = { + 'LAZY_RENDERING': False, +} \ No newline at end of file diff --git a/manage.py b/manage.py new file mode 100755 index 0000000000..8e7ac79b95 --- /dev/null +++ b/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/templates/create.html b/templates/create.html new file mode 100644 index 0000000000..80a31247fd --- /dev/null +++ b/templates/create.html @@ -0,0 +1,11 @@ + +
+ + + {% csrf_token %} + + + {{ form.as_p }} + + +
\ No newline at end of file From 864b647edc4be16ce9307e0a6d4ba8f878b70050 Mon Sep 17 00:00:00 2001 From: MiqSA Date: Sun, 30 Oct 2022 11:05:47 -0300 Subject: [PATCH 05/13] Update upload files --- apps/dataexplore/forms.py | 1 + apps/dataexplore/models.py | 7 +++--- apps/dataexplore/tests.py | 3 --- apps/dataexplore/views.py | 6 ++--- apps/users/tests.py | 3 --- config/settings.py | 47 +++++++++++++++++++------------------- config/urls.py | 7 ++++++ templates/create.html | 16 ++++++------- templates/home.html | 24 +++++++++++++++++++ 9 files changed, 70 insertions(+), 44 deletions(-) delete mode 100644 apps/dataexplore/tests.py delete mode 100644 apps/users/tests.py create mode 100644 templates/home.html diff --git a/apps/dataexplore/forms.py b/apps/dataexplore/forms.py index 0bc5dc1fe5..d1f6655a4c 100644 --- a/apps/dataexplore/forms.py +++ b/apps/dataexplore/forms.py @@ -6,5 +6,6 @@ class DataForm(forms.ModelForm): class Meta: model = DataModel fields = [ + "description", "filename", ] \ No newline at end of file diff --git a/apps/dataexplore/models.py b/apps/dataexplore/models.py index fe4ecd3d20..af83662019 100644 --- a/apps/dataexplore/models.py +++ b/apps/dataexplore/models.py @@ -1,11 +1,10 @@ from django.db import models -from datetime import datetime class DataModel(models.Model): """This class define the data model.""" - filename = models.CharField(max_length=255) - date_created = models.DateTimeField(default=datetime.now) - last_update = models.DateTimeField(default=datetime.now, blank=True) + description = models.CharField(max_length=255, blank=True) + filename = models.FileField(upload_to='files/', unique=True) + date_created = models.DateTimeField(auto_now_add=True) def __str__(self) -> str: return str(self.filename) diff --git a/apps/dataexplore/tests.py b/apps/dataexplore/tests.py deleted file mode 100644 index 7ce503c2dd..0000000000 --- a/apps/dataexplore/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/apps/dataexplore/views.py b/apps/dataexplore/views.py index fdd84d8d1f..6839a17198 100644 --- a/apps/dataexplore/views.py +++ b/apps/dataexplore/views.py @@ -1,6 +1,7 @@ from django.shortcuts import render from drf_yasg.utils import swagger_auto_schema from rest_framework.decorators import api_view +import os from .forms import DataForm @@ -9,11 +10,11 @@ TODO: Create the view to handle with divisions informations """ -@api_view(['POST']) +# @api_view(['POST']) def upload_files(request): # pylint: disable=unused-argument """This view make the upload files.""" context = {} - form = DataForm(request.POST or None) + form = DataForm(request.POST, request.FILES) if form.is_valid(): form.save() context['form'] = form @@ -29,4 +30,3 @@ def dashboard(): """This views brings informations about the transactions.""" pass - diff --git a/apps/users/tests.py b/apps/users/tests.py deleted file mode 100644 index 7ce503c2dd..0000000000 --- a/apps/users/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/config/settings.py b/config/settings.py index fd687ab59b..ee30d25c29 100644 --- a/config/settings.py +++ b/config/settings.py @@ -1,3 +1,6 @@ + +""" In this module the project settings are registreds.""" + from pathlib import Path import os from dotenv import load_dotenv @@ -5,10 +8,10 @@ load_dotenv() BASE_DIR = Path(__file__).resolve().parent.parent - SECRET_KEY = os.getenv('SECRET_KEY', 'MY_SECRET_KEY') DEVELOP_LOCAL = True +# SECURITY WARNING: don't run with debug turned on in production! DEBUG = True ALLOWED_HOSTS = [] @@ -23,7 +26,6 @@ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', - 'drf_yasg', 'django.contrib.sites', 'rest_framework', 'rest_framework.authtoken', @@ -32,11 +34,14 @@ 'allauth.account', 'dj_rest_auth.registration', 'allauth.socialaccount', + 'drf_yasg', 'corsheaders', - 'apps.dataexplore', 'apps.users', + 'apps.dataexplore', ] +SITE_ID = 1 + MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', @@ -68,9 +73,7 @@ WSGI_APPLICATION = 'config.wsgi.application' - - -if DEVELOP_LOCAL == True: +if DEVELOP_LOCAL: DATABASES = { 'default': { 'ENGINE': os.getenv('Engine'), @@ -101,9 +104,6 @@ 'django.contrib.auth.hashers.ScryptPasswordHasher', ] - - - AUTH_PASSWORD_VALIDATORS = [ { 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', @@ -120,7 +120,6 @@ ] - LANGUAGE_CODE = 'en-us' TIME_ZONE = 'UTC' @@ -130,15 +129,20 @@ USE_TZ = True - STATIC_URL = 'static/' - DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' +REST_SESSION_LOGIN = True EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' +SITE_ID = 1 +ACCOUNT_EMAIL_REQUIRED = False +ACCOUNT_AUTHENTICATION_METHOD = 'username' +ACCOUNT_EMAIL_VERIFICATION = 'optional' +REST_USE_JWT = True +JWT_AUTH_COOKIE = 'auth' REST_FRAMEWORK = { 'DEFAULT_AUTHENTICATION_CLASSES': ( @@ -149,18 +153,15 @@ 'DEFAULT_SCHEMA_CLASS': 'rest_framework.schemas.coreapi.AutoSchema' } +# SWAGGER_SETTINGS = { +# 'LOGIN_URL': 'login', +# 'LOGOUT_URL': 'logout', +# } +# For demo purposes only. Use a white list in the real world. CORS_ORIGIN_ALLOW_ALL = True +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' -SWAGGER_SETTINGS = { - 'SECURITY_DEFINITIONS': { - 'basic': { - 'type': 'basic' - } - }, -} - -REDOC_SETTINGS = { - 'LAZY_RENDERING': False, -} \ No newline at end of file +MEDIA_URL = '/media/' +MEDIA_ROOT = os.path.join(BASE_DIR, 'media') diff --git a/config/urls.py b/config/urls.py index 6ddae56352..992ed61c6b 100644 --- a/config/urls.py +++ b/config/urls.py @@ -1,5 +1,8 @@ from django.contrib import admin from django.urls import path, include, re_path +from django.conf import settings +from django.conf.urls.static import static +from django.views.generic.base import TemplateView from rest_framework import permissions from drf_yasg.views import get_schema_view from drf_yasg import openapi @@ -14,6 +17,7 @@ ) urlpatterns = [ + path('', TemplateView.as_view(template_name='home.html'), name='home'), path('admin/', admin.site.urls, name='admin'), path('users/', include('apps.users.urls'), name='users'), path('dataexplore/', include('apps.dataexplore.urls'), name='dataexplore'), @@ -21,3 +25,6 @@ re_path(r'^swagger/$', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'), re_path(r'^redoc/$', schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc'), ] + +if settings.DEBUG: + urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) \ No newline at end of file diff --git a/templates/create.html b/templates/create.html index 80a31247fd..d2d44fbc38 100644 --- a/templates/create.html +++ b/templates/create.html @@ -1,11 +1,11 @@ +{% extends 'rest_framework/base.html' %} -
- - +{% block content %} + {% csrf_token %} - - {{ form.as_p }} - - -
\ No newline at end of file + + + +

Return to home

+{% endblock %} \ No newline at end of file diff --git a/templates/home.html b/templates/home.html new file mode 100644 index 0000000000..9e425ed4d2 --- /dev/null +++ b/templates/home.html @@ -0,0 +1,24 @@ +{% extends 'rest_framework/base.html' %} + +{% block title %}Home{% endblock %} + +{% block content %} +

Home

+ +

Logout

+ + Acesse as informações do PIB +

Upload

+ + Acesse as informações sobre Estabelecimentos +

Dashboard

+ + Acesse a documentação da API no Swagger +

Documentação usando Swagger

+ + Acesse a documentação da API no Redoc +

Documentação Redoc

+ + + +{% endblock %} \ No newline at end of file From 8e9b268b80e7351d5bd9c9fb96e11ffc9d2e0ee2 Mon Sep 17 00:00:00 2001 From: MiqSA Date: Mon, 31 Oct 2022 08:45:31 -0300 Subject: [PATCH 06/13] Add data-explore unit test --- apps/__init__.py | 0 apps/dataexplore/tests/__int__.py | 0 apps/dataexplore/tests/test_e2e.py | 35 ++++++++++++++++++++++++++++++ apps/dataexplore/views.py | 20 +++++++---------- pytest.ini | 3 +++ requirements.txt | 2 ++ 6 files changed, 48 insertions(+), 12 deletions(-) create mode 100644 apps/__init__.py create mode 100644 apps/dataexplore/tests/__int__.py create mode 100644 apps/dataexplore/tests/test_e2e.py create mode 100644 pytest.ini diff --git a/apps/__init__.py b/apps/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/dataexplore/tests/__int__.py b/apps/dataexplore/tests/__int__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/apps/dataexplore/tests/test_e2e.py b/apps/dataexplore/tests/test_e2e.py new file mode 100644 index 0000000000..42f68af469 --- /dev/null +++ b/apps/dataexplore/tests/test_e2e.py @@ -0,0 +1,35 @@ +"""Here the end-to-end test data explore endpoint.""" +import io +from django.test import TestCase +from rest_framework.test import APIClient + + +class TestDataExplore(TestCase): + """In this class the data explore endpoints are tested.""" + def setUp(self): + """This function customize the setup test.""" + self.default_url = '/dataexplore/' + self.client = APIClient() + + def test_success_upload_file(self): + """This function test the success of upload files.""" + url = f'{self.default_url}upload-files' + data = { + 'filename': (io.BytesIO(b"Some data"), "fake-text-file.txt"), + 'description': 'This is a file text.' + } + + response = self.client.post(url, data=data, format='multipart') + assert response.status_code == 200 + assert response.context['data'] == 'Your file has been saved!' + + def test_fail_upload_file(self): + """This function test the success of upload files.""" + url = f'{self.default_url}upload-files' + data = { + 'description': 'This is a file text.' + } + + response = self.client.post(url, data=data, format='multipart') + assert response.status_code == 200 + assert response.context['data'] == 'Error in to save file.' diff --git a/apps/dataexplore/views.py b/apps/dataexplore/views.py index 6839a17198..18c29badc3 100644 --- a/apps/dataexplore/views.py +++ b/apps/dataexplore/views.py @@ -1,32 +1,28 @@ +"""Here the data explore views are created.""" from django.shortcuts import render -from drf_yasg.utils import swagger_auto_schema from rest_framework.decorators import api_view -import os +from apps.dataexplore.forms import DataForm -from .forms import DataForm -# Create your views here. -""" -TODO: Create the view to handle with divisions informations -""" - -# @api_view(['POST']) +@api_view(['POST']) def upload_files(request): # pylint: disable=unused-argument """This view make the upload files.""" context = {} form = DataForm(request.POST, request.FILES) if form.is_valid(): + context['data'] = 'Your file has been saved!' form.save() + else: + context['data'] = 'Error in to save file.' context['form'] = form return render(request, "create.html", context) + @api_view(['GET']) def data_transform(): """This view transform the data from file in informations to database.""" - pass + @api_view(['GET']) def dashboard(): """This views brings informations about the transactions.""" - pass - diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000000..ee7a0ba050 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +DJANGO_SETTINGS_MODULE=config.settings +python_files=tests.py test_*.py *_tests.py \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index f3656ce78b..eccc34f0f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,6 +40,7 @@ PyJWT==2.6.0 pylint==2.15.5 pyparsing==3.0.9 pytest==7.1.3 +pytest-django==4.5.2 python-dotenv==0.21.0 python3-openid==3.2.0 pytz==2022.5 @@ -48,6 +49,7 @@ requests-oauthlib==1.3.1 ruamel.yaml==0.17.21 ruamel.yaml.clib==0.2.7 sqlparse==0.4.3 +toml==0.10.2 tomli==2.0.1 tomlkit==0.11.5 uritemplate==4.1.1 From 6241f45e4212c364afa9ead49c4c6b0c1ebe8f9c Mon Sep 17 00:00:00 2001 From: MiqSA Date: Tue, 1 Nov 2022 11:15:01 -0300 Subject: [PATCH 07/13] Add dashboard template --- media/files/__init__.py | 0 requirements.txt | 6 ++++++ templates/dashboard.html | 25 +++++++++++++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 media/files/__init__.py create mode 100644 templates/dashboard.html diff --git a/media/files/__init__.py b/media/files/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/requirements.txt b/requirements.txt index eccc34f0f4..5eedca500c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,8 +27,11 @@ lazy-object-proxy==1.7.1 MarkupSafe==2.1.1 mccabe==0.7.0 mysqlclient==2.1.1 +numpy==1.23.4 oauthlib==3.2.2 packaging==21.3 +pandas==1.5.1 +Pillow==9.3.0 platformdirs==2.5.2 pluggy==1.0.0 py==1.11.0 @@ -41,13 +44,16 @@ pylint==2.15.5 pyparsing==3.0.9 pytest==7.1.3 pytest-django==4.5.2 +python-dateutil==2.8.2 python-dotenv==0.21.0 python3-openid==3.2.0 pytz==2022.5 +PyYAML==6.0 requests==2.28.1 requests-oauthlib==1.3.1 ruamel.yaml==0.17.21 ruamel.yaml.clib==0.2.7 +six==1.16.0 sqlparse==0.4.3 toml==0.10.2 tomli==2.0.1 diff --git a/templates/dashboard.html b/templates/dashboard.html new file mode 100644 index 0000000000..1cce79bce1 --- /dev/null +++ b/templates/dashboard.html @@ -0,0 +1,25 @@ +{% extends 'rest_framework/base.html' %} +{% block title %} Stores Manager {% endblock %} +{% block content %} + +

Valor Movimentado por Loja

+
+ + + + + + + {% for stores in results %} + + + + + + {% endfor %} +
LojaTransaçãoTotal
{{ stores.store_name }}{{ stores.transaction }}{{ stores.total_transaction }}
+

Return to home

+
+{% endblock %} From f29a1dcd1fc3b202d3ef9339f891e6126fd32290 Mon Sep 17 00:00:00 2001 From: MiqSA Date: Tue, 1 Nov 2022 11:29:20 -0300 Subject: [PATCH 08/13] Update app data-explore --- apps/dataexplore/fixtures/transactions.json | 1 + apps/dataexplore/manage_files.py | 47 ++++++++++ apps/dataexplore/models.py | 26 +++++- apps/dataexplore/serializers.py | 15 ++++ apps/dataexplore/tests/test_e2e.py | 16 ++-- apps/dataexplore/urls.py | 1 - apps/dataexplore/views.py | 95 +++++++++++++++++++-- apps/users/views.py | 12 --- config/settings.py | 4 +- 9 files changed, 184 insertions(+), 33 deletions(-) create mode 100644 apps/dataexplore/fixtures/transactions.json create mode 100644 apps/dataexplore/manage_files.py create mode 100644 apps/dataexplore/serializers.py diff --git a/apps/dataexplore/fixtures/transactions.json b/apps/dataexplore/fixtures/transactions.json new file mode 100644 index 0000000000..cf2bab18b0 --- /dev/null +++ b/apps/dataexplore/fixtures/transactions.json @@ -0,0 +1 @@ +[{"model": "dataexplore.transactiontypemodel", "pk": 1, "fields": {"name": "Débito", "mode": "Entrada", "signal": "+"}}, {"model": "dataexplore.transactiontypemodel", "pk": 2, "fields": {"name": "Boleto", "mode": "Saída", "signal": "-"}}, {"model": "dataexplore.transactiontypemodel", "pk": 3, "fields": {"name": "Financiamento", "mode": "Saída", "signal": "-"}}, {"model": "dataexplore.transactiontypemodel", "pk": 4, "fields": {"name": "Crédito", "mode": "Entrada", "signal": "+"}}, {"model": "dataexplore.transactiontypemodel", "pk": 5, "fields": {"name": "Recebimento Empréstimo", "mode": "Entrada", "signal": "+"}}, {"model": "dataexplore.transactiontypemodel", "pk": 6, "fields": {"name": "Vendas", "mode": "Entrada", "signal": "+"}}, {"model": "dataexplore.transactiontypemodel", "pk": 7, "fields": {"name": "Recebimento TED", "mode": "Entrada", "signal": "+"}}, {"model": "dataexplore.transactiontypemodel", "pk": 8, "fields": {"name": "Recebimento DOC", "mode": "Entrada", "signal": "+"}}, {"model": "dataexplore.transactiontypemodel", "pk": 9, "fields": {"name": "Aluguel", "mode": "Saída", "signal": "-"}}] \ No newline at end of file diff --git a/apps/dataexplore/manage_files.py b/apps/dataexplore/manage_files.py new file mode 100644 index 0000000000..164024bf4f --- /dev/null +++ b/apps/dataexplore/manage_files.py @@ -0,0 +1,47 @@ + + +class Information(): + def __init__(self, row=None): + self.row = row + + def fields_map(self): + try: + information_map = { + 'transaction': int(self.row[0]), + 'transaction_occurrence_date': f'{self.row[1:5]}-{self.row[5:7]}-{self.row[7:9]}', + 'transaction_value': int(self.row[9:19]) / 100, + 'client_cpf': self.row[19:30], + 'client_credit_card': self.row[30:42], + 'transaction_hour': int(self.row[42:48]), + 'store_owner': self.row[48:62], + 'store_name': self.row[62:82] + } + return information_map + except Exception as error: + return error + + + @staticmethod + def get_standard_transactions_type(): + description = ["Débito", "Boleto", "Financiamento", + "Crédito", "Recebimento Empréstimo", + "Vendas", "Recebimento TED", "Recebimento DOC", + "Aluguel"] + + mode = ["Entrada", "Saída", "Saída", "Entrada", + "Entrada", "Entrada", "Entrada", "Entrada", "Saída"] + + signal = ["+", "-", "-", "+", "+", "+", "+", "+", "-"] + + dict_type_transactions = [] + for ind, i in enumerate(description): + res = {} + fields = {} + res['model'] = 'dataexplore.transactiontypemodel' + res['pk'] = ind+1 + fields['name'] = i + fields['mode'] = mode[ind] + fields['signal'] = signal[ind] + res['fields'] = fields + dict_type_transactions.append(res) + return dict_type_transactions diff --git a/apps/dataexplore/models.py b/apps/dataexplore/models.py index af83662019..db275384a9 100644 --- a/apps/dataexplore/models.py +++ b/apps/dataexplore/models.py @@ -1,5 +1,7 @@ +"""Here the data-explore models are defined.""" from django.db import models + class DataModel(models.Model): """This class define the data model.""" description = models.CharField(max_length=255, blank=True) @@ -9,8 +11,26 @@ class DataModel(models.Model): def __str__(self) -> str: return str(self.filename) +class TransactionTypeModel(models.Model): + """This class define the transaction type model.""" + name = models.CharField(max_length=50, unique=True) + mode = models.CharField(max_length=20, blank=True) + signal = models.CharField(max_length=1, blank=True) + + def __str__(self) -> str: + return str(self.name) + class TransactionModel(models.Model): - pass + """This class define the transaction model.""" + transaction = models.ForeignKey(TransactionTypeModel, on_delete=models.CASCADE, related_name='transaction') + transaction_occurrence_date = models.CharField(max_length=10, blank=True) + transaction_value = models.FloatField(default=0.0) + client_cpf = models.CharField(max_length=11, blank=True) + client_credit_card = models.CharField(max_length=12, blank=True) + transaction_hour = models.IntegerField(default=0) + store_owner = models.CharField(max_length=250, blank=True) + store_name = models.CharField(max_length=250, blank=True) + + def __str__(self) -> str: + return str(self.transaction) -class TransactionTypeModel(models.Model): - pass diff --git a/apps/dataexplore/serializers.py b/apps/dataexplore/serializers.py new file mode 100644 index 0000000000..dd461497a5 --- /dev/null +++ b/apps/dataexplore/serializers.py @@ -0,0 +1,15 @@ +from rest_framework import serializers +from apps.dataexplore.models import TransactionModel, TransactionTypeModel + + + +class TransactionTypeSerializer(serializers.ModelSerializer): + class Meta: + model = TransactionTypeModel + fields = ['name', 'mode', 'signal'] + +class TransactionSerializer(serializers.ModelSerializer): + class Meta: + model = TransactionModel + fields = ['transaction', 'transaction_occurrence_date', 'transaction_value', 'client_cpf'] + diff --git a/apps/dataexplore/tests/test_e2e.py b/apps/dataexplore/tests/test_e2e.py index 42f68af469..fa9a901c10 100644 --- a/apps/dataexplore/tests/test_e2e.py +++ b/apps/dataexplore/tests/test_e2e.py @@ -1,7 +1,8 @@ -"""Here the end-to-end test data explore endpoint.""" +"""Here the end-to-end tests from data-explore endpoint.""" import io from django.test import TestCase from rest_framework.test import APIClient +import pytest class TestDataExplore(TestCase): @@ -10,21 +11,22 @@ def setUp(self): """This function customize the setup test.""" self.default_url = '/dataexplore/' self.client = APIClient() + - def test_success_upload_file(self): + def test_0_success_upload_file(self): """This function test the success of upload files.""" url = f'{self.default_url}upload-files' data = { - 'filename': (io.BytesIO(b"Some data"), "fake-text-file.txt"), + 'filename': (io.BytesIO(b"3201903010000014200096206760174753****3153153453JOAO MACEDO BAR DO JOAO "), "fake-text-file.txt"), 'description': 'This is a file text.' } response = self.client.post(url, data=data, format='multipart') - assert response.status_code == 200 - assert response.context['data'] == 'Your file has been saved!' + assert response.status_code == 302 + assert response.url == '/' - def test_fail_upload_file(self): - """This function test the success of upload files.""" + def test_1_fail_upload_file(self): + """This function test the fail of upload files.""" url = f'{self.default_url}upload-files' data = { 'description': 'This is a file text.' diff --git a/apps/dataexplore/urls.py b/apps/dataexplore/urls.py index 245289fb94..6e53c16a52 100644 --- a/apps/dataexplore/urls.py +++ b/apps/dataexplore/urls.py @@ -6,6 +6,5 @@ urlpatterns = [ path('upload-files', views.upload_files, name='upload-files'), - path('transform', views.data_transform, name='transform-data'), path('dashboard', views.dashboard, name='dashboard'), ] \ No newline at end of file diff --git a/apps/dataexplore/views.py b/apps/dataexplore/views.py index 18c29badc3..755ab37da0 100644 --- a/apps/dataexplore/views.py +++ b/apps/dataexplore/views.py @@ -1,17 +1,94 @@ """Here the data explore views are created.""" +import os +import pandas as pd from django.shortcuts import render from rest_framework.decorators import api_view from apps.dataexplore.forms import DataForm +from config.settings import MEDIA_ROOT +from django.http import HttpResponseRedirect, HttpResponse +from config.settings import MEDIA_ROOT +from rest_framework.response import Response +from apps.dataexplore.manage_files import Information +from rest_framework import status +from django.db.models import Sum +from apps.dataexplore.models import TransactionModel, TransactionTypeModel +from apps.dataexplore.serializers import TransactionSerializer + + + + +class TransformData(): + """This class transform the data from file in informations to database.""" + def __init__(self, filename): + self.filename = f'{MEDIA_ROOT}/{filename}' + self.file = 'media/'+filename + + def save_in_database(self): + """This function save informations in database.""" + + def get_informations(self, file): + for id_row in file.index: + row = file[0][id_row] + information = Information(row).fields_map() + + transaction_id = information['transaction'] + transaction_type = TransactionTypeModel.objects.get(pk=transaction_id) + + transaction = TransactionSerializer(instance=transaction_type, data=information) + if transaction.is_valid(): + information.pop('transaction') + obj, created = TransactionModel.objects.get_or_create(transaction=transaction_type, **information) + + result = (TransactionModel.objects + .values('store_name', 'transaction') + .annotate(total_transaction=Sum('transaction_value')) + .order_by('store_name') + ) + + for transaction_id in result: + transaction_type = TransactionTypeModel.objects.get(pk=transaction_id['transaction']) + transaction_id.pop('transaction') + transaction_id['transaction'] = transaction_type.name + return result + + + def read_file(self): + """This function read the txt file.""" + try: + file = pd.read_csv(self.filename, header=None) + return file + except Exception as err: + print('Error in read file', err) + return 500 + + def delete_file(self): + """This function delete the txt file.""" + if os.path.exists(self.file): + os.remove(self.file) + + def __call__(self): + file = self.read_file() + result = self.get_informations(file) + return result + + def main(self): + """This main function.""" + return self.__call__() + +def handle_uploaded_file(f): + with open('media/files/data.txt', 'wb+') as destination: + for chunk in f.chunks(): + destination.write(chunk) -@api_view(['POST']) def upload_files(request): # pylint: disable=unused-argument """This view make the upload files.""" context = {} - form = DataForm(request.POST, request.FILES) + form = DataForm(request.POST, request.FILES) # if form.is_valid(): context['data'] = 'Your file has been saved!' - form.save() + handle_uploaded_file(request.FILES['filename']) + return HttpResponseRedirect('/') else: context['data'] = 'Error in to save file.' context['form'] = form @@ -19,10 +96,10 @@ def upload_files(request): # pylint: disable=unused-argument @api_view(['GET']) -def data_transform(): - """This view transform the data from file in informations to database.""" - - -@api_view(['GET']) -def dashboard(): +def dashboard(request): """This views brings informations about the transactions.""" + result = TransformData(filename='files/data.txt').main() + if result != 500: + total = {'results': result} + return render(request, "dashboard.html", total) + return Response(status=status.HTTP_404_NOT_FOUND) diff --git a/apps/users/views.py b/apps/users/views.py index 8a4f670710..a321b5f0a1 100644 --- a/apps/users/views.py +++ b/apps/users/views.py @@ -6,8 +6,6 @@ from rest_framework import status, serializers from apps.users.serializers import UserSerializer -# from .forms import UsersForm - @api_view(['POST']) def create_user(request): @@ -52,13 +50,3 @@ def delete_user(request, user_id): # pylint: disable=unused-argument user = get_object_or_404(User, pk=user_id) user.delete() return Response(status=status.HTTP_202_ACCEPTED) - - -# def create_view(request): # pylint: disable=unused-argument -# """This view create user by UsersForm.""" -# context = {} -# form = UsersForm(request.POST or None) -# if form.is_valid(): -# form.save() -# context['form'] = form -# return render(request, "create.html", context) diff --git a/config/settings.py b/config/settings.py index ee30d25c29..88948b8941 100644 --- a/config/settings.py +++ b/config/settings.py @@ -14,7 +14,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = [ '0.0.0.0'] # Application definition @@ -165,3 +165,5 @@ MEDIA_URL = '/media/' MEDIA_ROOT = os.path.join(BASE_DIR, 'media') + +FIXTURE_DIRS = ['apps.dataexplore.fixtures'] From 89bdc8d93657010779c692d12b5e89b69de895d4 Mon Sep 17 00:00:00 2001 From: MiqSA Date: Tue, 1 Nov 2022 11:29:50 -0300 Subject: [PATCH 09/13] Update pytest config --- pytest.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index ee7a0ba050..df89ff65e1 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,4 @@ [pytest] DJANGO_SETTINGS_MODULE=config.settings -python_files=tests.py test_*.py *_tests.py \ No newline at end of file +python_files=tests.py test_*.py *_tests.py +addopts = -p no:warnings \ No newline at end of file From d0760684502991c08be543e39b0c49e09b78bb48 Mon Sep 17 00:00:00 2001 From: MiqSA Date: Tue, 1 Nov 2022 11:33:50 -0300 Subject: [PATCH 10/13] Update README --- README.md | 118 +++++++++++++++++++++++++----------------------------- 1 file changed, 55 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index c2bfac4078..ff2eedf9ac 100755 --- a/README.md +++ b/README.md @@ -1,85 +1,77 @@ -# Desafio programação - para vaga desenvolvedor +# Store Manager -Por favor leiam este documento do começo ao fim, com muita atenção. -O intuito deste teste é avaliar seus conhecimentos técnicos em programação. -O teste consiste em parsear [este arquivo de texto(CNAB)](https://github.com/ByCodersTec/desafio-ruby-on-rails/blob/master/CNAB.txt) e salvar suas informações(transações financeiras) em uma base de dados a critério do candidato. -Este desafio deve ser feito por você em sua casa. Gaste o tempo que você quiser, porém normalmente você não deve precisar de mais do que algumas horas. +Esse projeto consiste em parsear [este arquivo de texto(CNAB)](https://github.com/ByCodersTec/desafio-ruby-on-rails/blob/master/CNAB.txt) e salvar suas informações(transações financeiras) em uma base de dados. -# Instruções de entrega do desafio +# Sobre -1. Primeiro, faça um fork deste projeto para sua conta no Github (crie uma se você não possuir). -2. Em seguida, implemente o projeto tal qual descrito abaixo, em seu clone local. -3. Por fim, envie via email o projeto ou o fork/link do projeto para seu contato Bycoders_ com cópia para rh@bycoders.com.br. +Esse projeto é composto por uma interface web que aceita upload do [arquivo CNAB](https://github.com/ByCodersTec/desafio-ruby-on-rails/blob/master/CNAB.txt) (que contêm informações sobre movimentações financeiras de várias lojas), normaliza os dados e armazena em um banco de dados relacional MySQL. As informações das movimentações são exibidas na interface web. -# Descrição do projeto -Você recebeu um arquivo CNAB com os dados das movimentações finanaceira de várias lojas. -Precisamos criar uma maneira para que estes dados sejam importados para um banco de dados. +# Tecnologias Utilizadas -Sua tarefa é criar uma interface web que aceite upload do [arquivo CNAB](https://github.com/ByCodersTec/desafio-ruby-on-rails/blob/master/CNAB.txt), normalize os dados e armazene-os em um banco de dados relacional e exiba essas informações em tela. +## Back-end -**Sua aplicação web DEVE:** +- [Python](https://www.python.org/) +- [Django](https://www.djangoproject.com/) +- [Django Rest Framework](https://www.django-rest-framework.org/) +- [MySQL](https://www.mysql.com/) -1. Ter uma tela (via um formulário) para fazer o upload do arquivo(pontos extras se não usar um popular CSS Framework ) -2. Interpretar ("parsear") o arquivo recebido, normalizar os dados, e salvar corretamente a informação em um banco de dados relacional, **se atente as documentações** que estão logo abaixo. -3. Exibir uma lista das operações importadas por lojas, e nesta lista deve conter um totalizador do saldo em conta -4. Ser escrita na sua linguagem de programação de preferência -5. Ser simples de configurar e rodar, funcionando em ambiente compatível com Unix (Linux ou Mac OS X). Ela deve utilizar apenas linguagens e bibliotecas livres ou gratuitas. -6. Git com commits atomicos e bem descritos -7. PostgreSQL, MySQL ou SQL Server -8. Ter testes automatizados -9. Docker compose (Pontos extras se utilizar) -10. Readme file descrevendo bem o projeto e seu setup -11. Incluir informação descrevendo como consumir o endpoint da API +## Front-end -**Sua aplicação web não precisa:** +- HTML +- [Jinja](https://jinja.palletsprojects.com/en/3.0.x/) -1. Lidar com autenticação ou autorização (pontos extras se ela fizer, mais pontos extras se a autenticação for feita via OAuth). -2. Ser escrita usando algum framework específico (mas não há nada errado em usá-los também, use o que achar melhor). -3. Documentação da api.(Será um diferencial e pontos extras se fizer) +## Testes -# Documentação do CNAB +- Testes de unidade = [Pytes](https://docs.python.org/3/library/unittest.html) +```bash +pytest -v +``` +- Testes de Padronização e Formatação +[Pylint](https://pypi.org/project/pylint/) +[Flake8](https://pypi.org/project/flake8/) -| Descrição do campo | Inicio | Fim | Tamanho | Comentário -| ------------- | ------------- | -----| ---- | ------ -| Tipo | 1 | 1 | 1 | Tipo da transação -| Data | 2 | 9 | 8 | Data da ocorrência -| Valor | 10 | 19 | 10 | Valor da movimentação. *Obs.* O valor encontrado no arquivo precisa ser divido por cem(valor / 100.00) para normalizá-lo. -| CPF | 20 | 30 | 11 | CPF do beneficiário -| Cartão | 31 | 42 | 12 | Cartão utilizado na transação -| Hora | 43 | 48 | 6 | Hora da ocorrência atendendo ao fuso de UTC-3 -| Dono da loja | 49 | 62 | 14 | Nome do representante da loja -| Nome loja | 63 | 81 | 19 | Nome da loja +```bash +pylint apps.users.views.py -# Documentação sobre os tipos das transações +flake8 apps.users.views.py +``` -| Tipo | Descrição | Natureza | Sinal | -| ---- | -------- | --------- | ----- | -| 1 | Débito | Entrada | + | -| 2 | Boleto | Saída | - | -| 3 | Financiamento | Saída | - | -| 4 | Crédito | Entrada | + | -| 5 | Recebimento Empréstimo | Entrada | + | -| 6 | Vendas | Entrada | + | -| 7 | Recebimento TED | Entrada | + | -| 8 | Recebimento DOC | Entrada | + | -| 9 | Aluguel | Saída | - | -# Avaliação +# Como Utilizar? -Seu projeto será avaliado de acordo com os seguintes critérios. +```bash +# Clonar esse repositório +$ https://github.com/MiqSA/desafio-dev.git -1. Sua aplicação preenche os requerimentos básicos? -2. Você documentou a maneira de configurar o ambiente e rodar sua aplicação? -3. Você seguiu as instruções de envio do desafio? -4. Qualidade e cobertura dos testes unitários. +# Entrar no pasta do projeto +$ cd desafio-dev -Adicionalmente, tentaremos verificar a sua familiarização com as bibliotecas padrões (standard libs), bem como sua experiência com programação orientada a objetos a partir da estrutura de seu projeto. +# Subir aplicação pelo docker +$ docker-compose up -# Referência +# Mude para branch de desenvolvimento +$ git checkout features -Este desafio foi baseado neste outro desafio: https://github.com/lschallenges/data-engineering +# Sincronize com o repositório +$ git pull ---- +# Garantir que o docker desktop está ativo -Boa sorte! +$ docker-compose up --build + +# A aplicação estará funcionando em http://0.0.0.0:8000/ + +``` + +# Observações + +Dados iniciais + +```bash +python manage.py loaddata transactions +``` + +# Melhorias +- Front-end com uso de CSS e JavaScript para melhorar experiência do usuário. +- Testes de carga devem ser efetuados. From b9ed0cedf6f88f57eb418afb1a147a36723e5dfd Mon Sep 17 00:00:00 2001 From: MiqSA Date: Tue, 1 Nov 2022 11:39:30 -0300 Subject: [PATCH 11/13] Add docker files --- config/settings.py | 3 +-- docker-compose.yml | 47 +++++++++++++++++++++++++++++++++++++++++++ docker/api.dockerfile | 7 +++++++ 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 docker-compose.yml create mode 100644 docker/api.dockerfile diff --git a/config/settings.py b/config/settings.py index 88948b8941..2c6e604a1a 100644 --- a/config/settings.py +++ b/config/settings.py @@ -4,12 +4,11 @@ from pathlib import Path import os from dotenv import load_dotenv - load_dotenv() BASE_DIR = Path(__file__).resolve().parent.parent SECRET_KEY = os.getenv('SECRET_KEY', 'MY_SECRET_KEY') -DEVELOP_LOCAL = True +DEVELOP_LOCAL = False # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000..4c5b0d7409 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,47 @@ +version: '3.9' + +services: + db: + container_name: Store_manager + image: postgres + restart: always + volumes: + - ./data/db:/var/lib/postgresql/data + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=postgres + ports: + - "5432:5432" + networks: + - production-network + expose: + - 5432 + + api: + build: + dockerfile: docker/api.dockerfile + context: . + container_name: api_store_manager + command: > + sh -c " python manage.py makemigrations + python manage.py migrate + python manage.py loaddata transactions + python manage.py runserver 0.0.0.0:8000" + ports: + - "8000:8000" + volumes: + - .:/code + environment: + - POSTGRES_NAME=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + depends_on: + - db + networks: + - production-network + restart: on-failure:20 + +networks: + production-network: + driver: bridge diff --git a/docker/api.dockerfile b/docker/api.dockerfile new file mode 100644 index 0000000000..7190339add --- /dev/null +++ b/docker/api.dockerfile @@ -0,0 +1,7 @@ +FROM python:3 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +WORKDIR /code +COPY requirements.txt /code/ +RUN pip install -r requirements.txt +COPY . /code/ \ No newline at end of file From 8ee641df956df971c088dffec26df3fa3b231442 Mon Sep 17 00:00:00 2001 From: MiqSA Date: Tue, 1 Nov 2022 12:01:16 -0300 Subject: [PATCH 12/13] Update requirements data-explore views --- apps/dataexplore/views.py | 63 +++++++++++++++++++-------------------- config/settings.py | 2 +- requirements.txt | 1 + 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/apps/dataexplore/views.py b/apps/dataexplore/views.py index 755ab37da0..134f1e3a7f 100644 --- a/apps/dataexplore/views.py +++ b/apps/dataexplore/views.py @@ -2,16 +2,15 @@ import os import pandas as pd from django.shortcuts import render -from rest_framework.decorators import api_view -from apps.dataexplore.forms import DataForm -from config.settings import MEDIA_ROOT -from django.http import HttpResponseRedirect, HttpResponse -from config.settings import MEDIA_ROOT +from django.http import HttpResponseRedirect +from django.db.models import Sum from rest_framework.response import Response -from apps.dataexplore.manage_files import Information from rest_framework import status +from rest_framework.decorators import api_view +from config.settings import MEDIA_ROOT -from django.db.models import Sum +from apps.dataexplore.forms import DataForm +from apps.dataexplore.manage_files import Information from apps.dataexplore.models import TransactionModel, TransactionTypeModel from apps.dataexplore.serializers import TransactionSerializer @@ -24,30 +23,28 @@ def __init__(self, filename): self.filename = f'{MEDIA_ROOT}/{filename}' self.file = 'media/'+filename - def save_in_database(self): + def save_in_database(self, file): """This function save informations in database.""" - - def get_informations(self, file): for id_row in file.index: - row = file[0][id_row] - information = Information(row).fields_map() - - transaction_id = information['transaction'] - transaction_type = TransactionTypeModel.objects.get(pk=transaction_id) - - transaction = TransactionSerializer(instance=transaction_type, data=information) - if transaction.is_valid(): - information.pop('transaction') - obj, created = TransactionModel.objects.get_or_create(transaction=transaction_type, **information) - - result = (TransactionModel.objects + row = file[0][id_row] + information = Information(row).fields_map() + transaction_id = information['transaction'] + transaction_type = TransactionTypeModel.objects.get(pk=transaction_id) # pylint: disable=no-member + transaction = TransactionSerializer(instance=transaction_type, data=information) + if transaction.is_valid(): + information.pop('transaction') + TransactionModel.objects.get_or_create(transaction=transaction_type, **information) # pylint: disable=no-member + + def get_informations(self): + """This function get informations from database.""" + result = (TransactionModel.objects # pylint: disable=no-member .values('store_name', 'transaction') .annotate(total_transaction=Sum('transaction_value')) .order_by('store_name') ) - for transaction_id in result: - transaction_type = TransactionTypeModel.objects.get(pk=transaction_id['transaction']) + transaction_type = TransactionTypeModel.objects.get( # pylint: disable=no-member + pk=transaction_id['transaction']) transaction_id.pop('transaction') transaction_id['transaction'] = transaction_type.name return result @@ -58,7 +55,7 @@ def read_file(self): try: file = pd.read_csv(self.filename, header=None) return file - except Exception as err: + except Exception as err: # pylint: disable=broad-except print('Error in read file', err) return 500 @@ -69,28 +66,30 @@ def delete_file(self): def __call__(self): file = self.read_file() - result = self.get_informations(file) + self.save_in_database(file) + result = self.get_informations() return result def main(self): """This main function.""" return self.__call__() -def handle_uploaded_file(f): +def handle_uploaded_file(file): + """This function save the data in media files.""" with open('media/files/data.txt', 'wb+') as destination: - for chunk in f.chunks(): + for chunk in file.chunks(): destination.write(chunk) def upload_files(request): # pylint: disable=unused-argument - """This view make the upload files.""" + """This function make the upload files.""" context = {} - form = DataForm(request.POST, request.FILES) # + form = DataForm(request.POST, request.FILES) if form.is_valid(): context['data'] = 'Your file has been saved!' handle_uploaded_file(request.FILES['filename']) return HttpResponseRedirect('/') - else: - context['data'] = 'Error in to save file.' + + context['data'] = 'Error in to save file.' context['form'] = form return render(request, "create.html", context) diff --git a/config/settings.py b/config/settings.py index 2c6e604a1a..ce9cdb049c 100644 --- a/config/settings.py +++ b/config/settings.py @@ -8,7 +8,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent SECRET_KEY = os.getenv('SECRET_KEY', 'MY_SECRET_KEY') -DEVELOP_LOCAL = False +DEVELOP_LOCAL = True # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True diff --git a/requirements.txt b/requirements.txt index 5eedca500c..363893f2f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,6 +34,7 @@ pandas==1.5.1 Pillow==9.3.0 platformdirs==2.5.2 pluggy==1.0.0 +psycopg2-binary==2.9.3 py==1.11.0 pycodestyle==2.9.1 pycparser==2.21 From 7b7357821166e096076f0919d038c9baa73c7b4d Mon Sep 17 00:00:00 2001 From: MiqSA Date: Tue, 1 Nov 2022 12:02:27 -0300 Subject: [PATCH 13/13] Update settings to use docker --- config/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/settings.py b/config/settings.py index ce9cdb049c..2c6e604a1a 100644 --- a/config/settings.py +++ b/config/settings.py @@ -8,7 +8,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent SECRET_KEY = os.getenv('SECRET_KEY', 'MY_SECRET_KEY') -DEVELOP_LOCAL = True +DEVELOP_LOCAL = False # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True