From 08c8628a81de6d23ef08661d0aee0cf730c0a8d5 Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 15:56:21 +0200 Subject: [PATCH 01/24] Upgrade factory-boy to 2.11.1 In addition to being up-to-date, this removes the need to have and pin `fake-factory` becauses uses the replacement `faker`. --- requirements/tests.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements/tests.txt b/requirements/tests.txt index 87f171e..a7e2f74 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -3,5 +3,4 @@ mock==2.0.0 coverage -factory-boy==2.7.0 -fake-factory==0.7.4 +factory-boy==2.11.1 From d637a48813bca8385db23627d50989c276f50f0e Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 16:00:12 +0200 Subject: [PATCH 02/24] Upgrade django-filter to 1.1.0 --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 22f6306..2ca2b9e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,7 +1,7 @@ celery==3.1.23 Django==1.10 django-celery==3.1.17 -django-filter==0.14.0 +django-filter==1.1.0 djangorestframework==3.4.5 django-rest-swagger==2.0.5 pycaption==1.0.0 From 962e08f2a8e4ab6d34c085622c7c3949826333f0 Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 15:57:55 +0200 Subject: [PATCH 03/24] Upgrade Django to last 1.10 version: 1.10.8 --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 2ca2b9e..16ff102 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ celery==3.1.23 -Django==1.10 +Django==1.10.8 django-celery==3.1.17 django-filter==1.1.0 djangorestframework==3.4.5 From 90d9d834014c2d885134b99eaf97da4ff9ff4be5 Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 16:02:19 +0200 Subject: [PATCH 04/24] Upgrade Django to last 1.11 version: 1.11.13 --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 16ff102..a5141bc 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ celery==3.1.23 -Django==1.10.8 +Django==1.11.13 django-celery==3.1.17 django-filter==1.1.0 djangorestframework==3.4.5 From 9d7ed6ee9270686af6a52cb412c5cea99995ab38 Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 16:07:38 +0200 Subject: [PATCH 05/24] Upgrade djangorestframework to last 3.6 version: 3.6.4 `3.6.4` is the last version working without changes. 3.7 needs change about using django-filter instead of previously integrated feature in drf. --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index a5141bc..44a39bc 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -2,7 +2,7 @@ celery==3.1.23 Django==1.11.13 django-celery==3.1.17 django-filter==1.1.0 -djangorestframework==3.4.5 +djangorestframework==3.6.4 django-rest-swagger==2.0.5 pycaption==1.0.0 Markdown==2.6.6 From c897a1f140747a612fdb4c202b756c04091d99fb Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 16:08:39 +0200 Subject: [PATCH 06/24] Upgrade django-rest-swagger to 2.2.0 --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 44a39bc..1238c77 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -3,7 +3,7 @@ Django==1.11.13 django-celery==3.1.17 django-filter==1.1.0 djangorestframework==3.6.4 -django-rest-swagger==2.0.5 +django-rest-swagger==2.2.0 pycaption==1.0.0 Markdown==2.6.6 Pillow==3.3.1 From e7251002c3578c99b733df6f219e13eeabd81af8 Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 16:10:17 +0200 Subject: [PATCH 07/24] Upgrade Markdown to 2.6.11 --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 1238c77..7957c01 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -5,5 +5,5 @@ django-filter==1.1.0 djangorestframework==3.6.4 django-rest-swagger==2.2.0 pycaption==1.0.0 -Markdown==2.6.6 +Markdown==2.6.11 Pillow==3.3.1 From 2529d3976e98204f9074cce1ca659ebd62fc2358 Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 16:11:14 +0200 Subject: [PATCH 08/24] Upgrade pycaption to 1.0.1 --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 7957c01..538e673 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,6 +4,6 @@ django-celery==3.1.17 django-filter==1.1.0 djangorestframework==3.6.4 django-rest-swagger==2.2.0 -pycaption==1.0.0 +pycaption==1.0.1 Markdown==2.6.11 Pillow==3.3.1 From f0014fbd3657bf73b6c3509d229be96a5d8a9ec3 Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 16:14:24 +0200 Subject: [PATCH 09/24] Upgrade Pillow to 5.1.0 --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 538e673..b23484e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,4 +6,4 @@ djangorestframework==3.6.4 django-rest-swagger==2.2.0 pycaption==1.0.1 Markdown==2.6.11 -Pillow==3.3.1 +Pillow==5.1.0 From 5ab011f4819feb3c8be4486a6d9947756091b207 Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 16:17:28 +0200 Subject: [PATCH 10/24] Upgrade boto3 to 1.7.35 --- requirements/aws.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/aws.txt b/requirements/aws.txt index 21c44a5..a31d86a 100644 --- a/requirements/aws.txt +++ b/requirements/aws.txt @@ -1 +1 @@ -boto3==1.3.1 +boto3==1.7.35 From 8f0f9c15c2b00b754de7c70a94306b204d1d7847 Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 16:31:54 +0200 Subject: [PATCH 11/24] Upgrade celery to last 3 version: 3.1.25 Before upgrading 4+, it was recommended to upgrade to 3.1.25. --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index b23484e..e6e41f7 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,4 +1,4 @@ -celery==3.1.23 +celery==3.1.25 Django==1.11.13 django-celery==3.1.17 django-filter==1.1.0 From cb04a81cda6b094f3b943afb8afcf8a6abc8c1a6 Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 16:32:16 +0200 Subject: [PATCH 12/24] Upgrade django-celery to 3.2.2 --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index e6e41f7..a9d10c3 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,6 +1,6 @@ celery==3.1.25 Django==1.11.13 -django-celery==3.1.17 +django-celery==3.2.2 django-filter==1.1.0 djangorestframework==3.6.4 django-rest-swagger==2.2.0 From 2c9c57cb033830b271774dc3fb54115a3d8c4b16 Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 16:57:14 +0200 Subject: [PATCH 13/24] Upgrade djangoresetframework to last 3.7 version: 3.77 `rest_framework.filters` is removed in `3.7` and replaced by `django_filters.rest_framework`. --- api/v1/views.py | 2 +- requirements/base.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/v1/views.py b/api/v1/views.py index e8fcddd..db7d705 100644 --- a/api/v1/views.py +++ b/api/v1/views.py @@ -3,7 +3,7 @@ from django.contrib.auth.models import User from django.db import transaction import django_filters -from rest_framework import filters +from django_filters import rest_framework as filters from rest_framework import mixins from rest_framework import status as rest_status from rest_framework import viewsets diff --git a/requirements/base.txt b/requirements/base.txt index a9d10c3..8ec0d8b 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -2,7 +2,7 @@ celery==3.1.25 Django==1.11.13 django-celery==3.2.2 django-filter==1.1.0 -djangorestframework==3.6.4 +djangorestframework==3.7.7 django-rest-swagger==2.2.0 pycaption==1.0.1 Markdown==2.6.11 From ba74458f129a3150426577a5efccfa60abd5078a Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 17:07:51 +0200 Subject: [PATCH 14/24] Upgrade djangoresetframework to 3.8.2 In DFR 3.8, it's not possible anymore to have read-only fields in serializers having a default value set on creation. For this, there is `CreateOnlyDefault`, so we use this for `VideoUploadUrlSerializer.expires_at` --- api/v1/serializers.py | 5 +++-- requirements/base.txt | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/api/v1/serializers.py b/api/v1/serializers.py index 5553e7c..9481558 100644 --- a/api/v1/serializers.py +++ b/api/v1/serializers.py @@ -41,8 +41,9 @@ def get_queryset(self): id = serializers.CharField(source='public_video_id', read_only=True) expires_at = serializers.IntegerField( - read_only=True, - default=lambda: time() + models.VideoUploadUrl.objects.EXPIRE_DELAY + default=serializers.CreateOnlyDefault( + lambda: time() + models.VideoUploadUrl.objects.EXPIRE_DELAY + ) ) owner = serializers.HiddenField(default=serializers.CurrentUserDefault()) playlist = RelatedPlaylistField(slug_field='public_id', required=False) diff --git a/requirements/base.txt b/requirements/base.txt index 8ec0d8b..5aff91d 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -2,7 +2,7 @@ celery==3.1.25 Django==1.11.13 django-celery==3.2.2 django-filter==1.1.0 -djangorestframework==3.7.7 +djangorestframework==3.8.2 django-rest-swagger==2.2.0 pycaption==1.0.1 Markdown==2.6.11 From c319e2a05de52c0e5864d0a0e53adba1399cc6fc Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 17:27:55 +0200 Subject: [PATCH 15/24] Add on_delete arg to FK/O2O fields They will be required in Django 2. The actual default is `CASCADE` so we set the arg to this value, it implies no changes. --- pipeline/models.py | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/pipeline/models.py b/pipeline/models.py index 6329555..9db9296 100644 --- a/pipeline/models.py +++ b/pipeline/models.py @@ -26,7 +26,7 @@ class Video(models.Model): default=utils.generate_long_random_id, ) - owner = models.ForeignKey(User) + owner = models.ForeignKey(User, on_delete=models.CASCADE) def __str__(self): return '{} - {}'.format(self.public_id, self.title) @@ -60,7 +60,7 @@ def create_video_processing_state(sender, instance=None, created=False, **kwargs class Playlist(models.Model): name = models.CharField(max_length=128, db_index=True) videos = models.ManyToManyField(Video, related_name='playlists') - owner = models.ForeignKey(User) + owner = models.ForeignKey(User, on_delete=models.CASCADE) public_id = models.CharField( max_length=20, unique=True, validators=[MinLengthValidator(1)], @@ -94,10 +94,15 @@ class VideoUploadUrl(models.Model): default=False, db_index=True ) - owner = models.ForeignKey(User, related_name='video_upload_urls') + owner = models.ForeignKey( + User, + related_name='video_upload_urls', + on_delete=models.CASCADE + ) playlist = models.ForeignKey( Playlist, verbose_name="Playlist to which the video will be added after upload", + on_delete=models.CASCADE, blank=True, null=True ) origin = models.CharField( @@ -127,7 +132,11 @@ class ProcessingState(models.Model): (STATUS_RESTART, 'Restart'), ) - video = models.OneToOneField(Video, related_name='processing_state') + video = models.OneToOneField( + Video, + related_name='processing_state', + on_delete=models.CASCADE + ) started_at = models.DateTimeField( verbose_name="Time of processing job start", auto_now=True @@ -152,7 +161,11 @@ def __str__(self): class Subtitle(models.Model): - video = models.ForeignKey(Video, related_name='subtitles') + video = models.ForeignKey( + Video, + related_name='subtitles', + on_delete=models.CASCADE + ) public_id = models.CharField( max_length=20, unique=True, validators=[MinLengthValidator(1)], @@ -179,7 +192,11 @@ def __str__(self): class VideoFormat(models.Model): - video = models.ForeignKey(Video, related_name='formats') + video = models.ForeignKey( + Video, + related_name='formats', + on_delete=models.CASCADE + ) name = models.CharField(max_length=128) bitrate = models.FloatField(validators=[MinValueValidator(0)]) From 408122b414c77dff6829279332bfb9e64515340e Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 17:32:12 +0200 Subject: [PATCH 16/24] Add app_name included urls To solve this that will break when migrating to Django 2 https://docs.djangoproject.com/en/dev/releases/1.9/#passing-a-3-tuple-or-an-app-name-to-include ```bash $ python -Wd manage.py check api/urls.py:4: RemovedInDjango20Warning: Specifying a namespace in django.conf.urls.include() without providing an app_name is deprecated. Set the app_name attribute in the included module, or pass a 2-tuple containing the list of patterns and app_name instead. url(r'^v1/', include('api.v1.urls', namespace="v1")), videofront/urls.py:8: RemovedInDjango20Warning: Specifying a namespace in django.conf.urls.include() without providing an app_name is deprecated. Set the app_name attribute in the included module, or pass a 2-tuple containing the list of patterns and app_name instead. url(r'^api/', include('api.urls', namespace="api")), ``` --- api/urls.py | 5 ++++- api/v1/urls.py | 3 +++ videofront/urls.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/api/urls.py b/api/urls.py index 8444faf..1657f67 100644 --- a/api/urls.py +++ b/api/urls.py @@ -1,5 +1,8 @@ from django.conf.urls import url, include + +app_name = "api" + urlpatterns = [ - url(r'^v1/', include('api.v1.urls', namespace="v1")), + url(r'^v1/', include('api.v1.urls')), ] diff --git a/api/v1/urls.py b/api/v1/urls.py index e428463..490a235 100644 --- a/api/v1/urls.py +++ b/api/v1/urls.py @@ -6,6 +6,9 @@ from . import views +app_name = "v1" + + class Router(routers.DefaultRouter): """ We override the router in order to provide some documentation to the API root. diff --git a/videofront/urls.py b/videofront/urls.py index 8b4bf96..d0c5a79 100644 --- a/videofront/urls.py +++ b/videofront/urls.py @@ -5,7 +5,7 @@ urlpatterns = [ url(r'^$', RedirectView.as_view(pattern_name='api:v1:api-root'), name='home'), - url(r'^api/', include('api.urls', namespace="api")), + url(r'^api/', include('api.urls')), url(r'^admin/', admin.site.urls), ] From 4c347257a2b2cca133fbf586b6090c70ffbc1f8c Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 17:43:20 +0200 Subject: [PATCH 17/24] Replace django.core.urlresolvers by from django.urls import reverse Will break in django 2 https://docs.djangoproject.com/en/2.0/releases/1.10/#id3 --- api/tests/v1/test_playlists.py | 2 +- api/tests/v1/test_subtitles.py | 2 +- api/tests/v1/test_users.py | 2 +- api/tests/v1/test_video_upload_urls.py | 2 +- api/tests/v1/test_videos.py | 2 +- api/tests/v1/test_views.py | 2 +- pipeline/tests/test_tasks.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/api/tests/v1/test_playlists.py b/api/tests/v1/test_playlists.py index 3d3ab73..3260341 100644 --- a/api/tests/v1/test_playlists.py +++ b/api/tests/v1/test_playlists.py @@ -1,4 +1,4 @@ -from django.core.urlresolvers import reverse +from django.urls import reverse from pipeline.tests import factories from .base import BaseAuthenticatedTests diff --git a/api/tests/v1/test_subtitles.py b/api/tests/v1/test_subtitles.py index d21e626..d714e8e 100644 --- a/api/tests/v1/test_subtitles.py +++ b/api/tests/v1/test_subtitles.py @@ -1,8 +1,8 @@ from __future__ import unicode_literals from io import StringIO -from django.core.urlresolvers import reverse from django.test.utils import override_settings +from django.urls import reverse from mock import Mock, patch from pipeline import models diff --git a/api/tests/v1/test_users.py b/api/tests/v1/test_users.py index aea4b98..7580aa3 100644 --- a/api/tests/v1/test_users.py +++ b/api/tests/v1/test_users.py @@ -1,5 +1,5 @@ from django.contrib.auth.models import User -from django.core.urlresolvers import reverse +from django.urls import reverse from .base import BaseAuthenticatedTests diff --git a/api/tests/v1/test_video_upload_urls.py b/api/tests/v1/test_video_upload_urls.py index 254a25d..e2335b4 100644 --- a/api/tests/v1/test_video_upload_urls.py +++ b/api/tests/v1/test_video_upload_urls.py @@ -1,7 +1,7 @@ from io import StringIO from time import time -from django.core.urlresolvers import reverse +from django.urls import reverse from mock import Mock from pipeline import models diff --git a/api/tests/v1/test_videos.py b/api/tests/v1/test_videos.py index ab193a0..9925743 100644 --- a/api/tests/v1/test_videos.py +++ b/api/tests/v1/test_videos.py @@ -2,9 +2,9 @@ import json from time import time -from django.core.urlresolvers import reverse from django.test import TestCase from django.test.utils import override_settings +from django.urls import reverse from django.utils.timezone import datetime, get_current_timezone from mock import Mock, patch diff --git a/api/tests/v1/test_views.py b/api/tests/v1/test_views.py index 0b866ff..296ccf3 100644 --- a/api/tests/v1/test_views.py +++ b/api/tests/v1/test_views.py @@ -1,6 +1,6 @@ from django.contrib.auth.models import User -from django.core.urlresolvers import reverse from django.test import TestCase +from django.urls import reverse class ApiV1Tests(TestCase): diff --git a/pipeline/tests/test_tasks.py b/pipeline/tests/test_tasks.py index 676c47b..6d73e7c 100644 --- a/pipeline/tests/test_tasks.py +++ b/pipeline/tests/test_tasks.py @@ -2,10 +2,10 @@ from time import time from mock import Mock -from django.core.urlresolvers import reverse from django.db.utils import IntegrityError from django.test import TestCase, TransactionTestCase from django.test.utils import override_settings +from django.urls import reverse from pipeline import exceptions from pipeline import models From 6bbfcd13841bdfe3a163bacdd6139834d101757c Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 17:56:35 +0200 Subject: [PATCH 18/24] Resolve django-filter deprecation http://django-filter.readthedocs.io/en/latest/guide/migration.html#filter-name-renamed-to-filter-field-name ``` $ python -Wd manage.py check api/v1/views.py:181: DeprecationWarning: `Filter.name` has been renamed to `Filter.field_name`. See: https://django-filter.readthedocs.io/en/develop/migration.html playlist_id = django_filters.CharFilter(name="playlists", lookup_expr="public_id") ``` --- api/v1/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/v1/views.py b/api/v1/views.py index db7d705..d6212a5 100644 --- a/api/v1/views.py +++ b/api/v1/views.py @@ -178,7 +178,7 @@ class VideoFilter(filters.FilterSet): """ Filter videos by playlist public id. """ - playlist_id = django_filters.CharFilter(name="playlists", lookup_expr="public_id") + playlist_id = django_filters.CharFilter(field_name="playlists", lookup_expr="public_id") class Meta: model = models.Video From bc6ea728c0cfa3b2bfb48f95852d4141df93d3eb Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 18:02:39 +0200 Subject: [PATCH 19/24] Use @action instead of @detail_route for DRF function views http://www.django-rest-framework.org/topics/3.8-announcement/#action-decorator-replaces-list_route-and-detail_route ``` $ python -Wd manage.py check api/v1/views.py:68: PendingDeprecationWarning: `detail_route` is pending deprecation and will be removed in 3.10 in favor of `action`, which accepts a `detail` bool. Use `@action(detail=True)` instead. @detail_route(methods=['POST']) api/v1/views.py:82: PendingDeprecationWarning: `detail_route` is pending deprecation and will be removed in 3.10 in favor of `action`, which accepts a `detail` bool. Use `@action(detail=True)` instead. @detail_route(methods=['POST']) api/v1/views.py:266: PendingDeprecationWarning: `detail_route` is pending deprecation and will be removed in 3.10 in favor of `action`, which accepts a `detail` bool. Use `@action(detail=True)` instead. @detail_route(methods=['POST']) api/v1/views.py:302: PendingDeprecationWarning: `detail_route` is pending deprecation and will be removed in 3.10 in favor of `action`, which accepts a `detail` bool. Use `@action(detail=True)` instead. @detail_route(methods=['POST']) api/v1/views.py:351: PendingDeprecationWarning: `detail_route` is pending deprecation and will be removed in 3.10 in favor of `action`, which accepts a `detail` bool. Use `@action(detail=True)` instead. @detail_route(methods=['POST', 'OPTIONS']) ``` --- api/v1/views.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/api/v1/views.py b/api/v1/views.py index d6212a5..429083c 100644 --- a/api/v1/views.py +++ b/api/v1/views.py @@ -8,7 +8,7 @@ from rest_framework import status as rest_status from rest_framework import viewsets from rest_framework.authentication import SessionAuthentication, BasicAuthentication, TokenAuthentication -from rest_framework.decorators import api_view, detail_route, renderer_classes +from rest_framework.decorators import action, api_view, renderer_classes from rest_framework.permissions import IsAuthenticated, IsAdminUser from rest_framework.response import Response from rest_framework.schemas import SchemaGenerator @@ -65,7 +65,7 @@ def get_queryset(self): return models.Playlist.objects.filter(owner=self.request.user) - @detail_route(methods=['POST']) + @action(methods=['post'], detail=True) def add_video(self, request, **kwargs): """ Add a video to a playlist @@ -79,7 +79,7 @@ def add_video(self, request, **kwargs): playlist.videos.add(video) return Response(status=rest_status.HTTP_204_NO_CONTENT) - @detail_route(methods=['POST']) + @action(methods=['post'], detail=True) def remove_video(self, request, **kwargs): """ Remove a video from a playlist @@ -263,7 +263,7 @@ def perform_destroy(self, instance): super(VideoViewSet, self).perform_destroy(instance) tasks.delete_video(instance.public_id) - @detail_route(methods=['POST']) + @action(methods=['post'], detail=True) def subtitles(self, request, **kwargs): """ Subtitle upload @@ -299,7 +299,7 @@ def subtitles(self, request, **kwargs): return Response(serializer.data, status=rest_status.HTTP_201_CREATED) - @detail_route(methods=['POST']) + @action(methods=['post'], detail=True) def thumbnail(self, request, **kwargs): """ Thumbnail upload @@ -348,7 +348,7 @@ class UploadViewset(viewsets.ViewSet): lookup_field = 'public_video_id' lookup_url_kwarg = 'video_id' - @detail_route(methods=['POST', 'OPTIONS']) + @action(methods=['post', 'options'], detail=True) def upload(self, request, video_id=None): """ Upload a video file. From a2927c43b927304f1991aee03f446756ac8ff5a9 Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 18:28:16 +0200 Subject: [PATCH 20/24] Use new middleware system from django 1.10 "session authentication" is enabled by default in django 1.10 so `SessionAuthenticationMiddleware` was not needed anymore. And removed in django 2. --- videofront/settings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/videofront/settings.py b/videofront/settings.py index 4ac5259..d0ce671 100644 --- a/videofront/settings.py +++ b/videofront/settings.py @@ -33,13 +33,12 @@ 'pipeline', ] -MIDDLEWARE_CLASSES = [ +MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] From 720733cb5e5ce1d4254dbd5d8ada73a5004128be Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Mon, 11 Jun 2018 18:29:03 +0200 Subject: [PATCH 21/24] Upgrade Django to 2.0.6 --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 5aff91d..369a60b 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ celery==3.1.25 -Django==1.11.13 +Django==2.0.6 django-celery==3.2.2 django-filter==1.1.0 djangorestframework==3.8.2 From 4404c8ad7704d8cf7b218fccd8895c820f62b3e6 Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Thu, 14 Jun 2018 15:49:02 +0200 Subject: [PATCH 22/24] Support only 3.6+ --- .travis.yml | 3 +-- README.md | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 078862b..3c98466 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: python python: - - "3.4" - - "3.5" + - "3.6" sudo: required dist: trusty before_install: diff --git a/README.md b/README.md index bed9437..7b79604 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Videofront [![Build Status](https://travis-ci.org/openfun/videofront.svg?branch=master)](https://travis-ci.org/openfun/videofront) -A scalable video hosting platform written in Django. +A scalable video hosting platform written in Django 2, for Python 3.6. Initially, Videofront was developed to host videos of MOOCs taught on [Open edX](https://open.edx.org/) platforms, but it can easily be used in just any web platform that requires video embedding. From 3214a443b5ff634f44d84a2879457b547013e5c2 Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Thu, 14 Jun 2018 15:35:12 +0200 Subject: [PATCH 23/24] `mock` is available from `unittest` in python 3 We use `assert_called_once` which is only present in python 3.6 --- api/tests/v1/test_subtitles.py | 2 +- api/tests/v1/test_video_upload_urls.py | 2 +- api/tests/v1/test_videos.py | 2 +- contrib/plugins/aws/tests/test_backend.py | 3 +-- pipeline/tests/test_tasks.py | 2 +- requirements/tests.txt | 1 - 6 files changed, 5 insertions(+), 7 deletions(-) diff --git a/api/tests/v1/test_subtitles.py b/api/tests/v1/test_subtitles.py index d714e8e..9fab706 100644 --- a/api/tests/v1/test_subtitles.py +++ b/api/tests/v1/test_subtitles.py @@ -1,9 +1,9 @@ from __future__ import unicode_literals from io import StringIO +from unittest.mock import Mock, patch from django.test.utils import override_settings from django.urls import reverse -from mock import Mock, patch from pipeline import models from pipeline.tests.utils import override_plugin_backend diff --git a/api/tests/v1/test_video_upload_urls.py b/api/tests/v1/test_video_upload_urls.py index e2335b4..12d617b 100644 --- a/api/tests/v1/test_video_upload_urls.py +++ b/api/tests/v1/test_video_upload_urls.py @@ -1,8 +1,8 @@ from io import StringIO from time import time +from unittest.mock import Mock from django.urls import reverse -from mock import Mock from pipeline import models from pipeline.tests.utils import override_plugin_backend diff --git a/api/tests/v1/test_videos.py b/api/tests/v1/test_videos.py index 9925743..1b3ceca 100644 --- a/api/tests/v1/test_videos.py +++ b/api/tests/v1/test_videos.py @@ -1,12 +1,12 @@ from io import BytesIO import json from time import time +from unittest.mock import Mock, patch from django.test import TestCase from django.test.utils import override_settings from django.urls import reverse from django.utils.timezone import datetime, get_current_timezone -from mock import Mock, patch from pipeline import models from pipeline.tests.utils import override_plugin_backend diff --git a/contrib/plugins/aws/tests/test_backend.py b/contrib/plugins/aws/tests/test_backend.py index 1e0cb31..de5c5d0 100644 --- a/contrib/plugins/aws/tests/test_backend.py +++ b/contrib/plugins/aws/tests/test_backend.py @@ -1,12 +1,11 @@ from io import BytesIO import shutil +from unittest.mock import Mock, patch from botocore.exceptions import ClientError from django.test import TestCase from django.test.utils import override_settings -from mock import Mock, patch - import pipeline.backend import pipeline.exceptions import pipeline.tasks diff --git a/pipeline/tests/test_tasks.py b/pipeline/tests/test_tasks.py index 6d73e7c..e36ae47 100644 --- a/pipeline/tests/test_tasks.py +++ b/pipeline/tests/test_tasks.py @@ -1,6 +1,6 @@ import os from time import time -from mock import Mock +from unittest.mock import Mock from django.db.utils import IntegrityError from django.test import TestCase, TransactionTestCase diff --git a/requirements/tests.txt b/requirements/tests.txt index a7e2f74..837959c 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,6 +1,5 @@ -r base.txt -r aws.txt -mock==2.0.0 coverage factory-boy==2.11.1 From fe0be1e965f8f8bc312de6fb603e343b10110058 Mon Sep 17 00:00:00 2001 From: "Stephane Angel (Twidi)" Date: Thu, 14 Jun 2018 15:49:19 +0200 Subject: [PATCH 24/24] Apply isort & black --- README.md | 6 +- api/apps.py | 2 +- api/models.py | 1 + api/tests/test_models.py | 4 +- api/tests/v1/base.py | 3 +- api/tests/v1/test_playlists.py | 58 ++- api/tests/v1/test_subtitles.py | 119 +++--- api/tests/v1/test_users.py | 49 +-- api/tests/v1/test_video_upload_urls.py | 89 +++-- api/tests/v1/test_videos.py | 231 +++++++----- api/tests/v1/test_views.py | 20 +- api/urls.py | 6 +- api/v1/serializers.py | 39 +- api/v1/urls.py | 28 +- api/v1/utils.py | 2 +- api/v1/views.py | 214 ++++++----- contrib/plugins/aws/apps.py | 2 +- contrib/plugins/aws/backend.py | 116 +++--- .../aws/management/commands/bootstrap-s3.py | 33 +- .../management/commands/delete-s3-folders.py | 6 +- contrib/plugins/aws/tests/test_backend.py | 222 ++++++----- contrib/plugins/aws/tests/utils.py | 14 +- manage.py | 1 + pipeline/admin.py | 51 +-- pipeline/apps.py | 2 +- pipeline/backend.py | 8 +- pipeline/cache.py | 3 + pipeline/exceptions.py | 4 + pipeline/management/commands/createuser.py | 26 +- .../management/commands/transcode-video.py | 8 +- pipeline/managers.py | 4 +- pipeline/migrations/0001_initial.py | 353 +++++++++++++++--- .../migrations/0002_auto_20160824_0640.py | 9 +- .../migrations/0003_auto_20160824_0733.py | 11 +- .../migrations/0004_auto_20160825_0741.py | 22 +- .../migrations/0005_auto_20160825_0819.py | 27 +- .../migrations/0006_auto_20160905_1245.py | 23 +- .../migrations/0007_auto_20160913_1247.py | 14 +- .../migrations/0008_videouploadurl_origin.py | 17 +- .../migrations/0009_auto_20160914_1204.py | 91 ++++- .../0010_video_public_thumbnail_id.py | 17 +- pipeline/migrations/0011_create_thumbnails.py | 7 +- .../migrations/0012_auto_20160915_1003.py | 18 +- .../migrations/0013_auto_20180124_0930.py | 106 +++++- pipeline/models.py | 118 +++--- pipeline/tasks.py | 89 +++-- pipeline/tests/factories.py | 5 +- pipeline/tests/test_backend.py | 1 - pipeline/tests/test_models.py | 26 +- pipeline/tests/test_tasks.py | 301 ++++++++------- pipeline/tests/test_utils.py | 16 +- pipeline/tests/utils.py | 1 + pipeline/utils.py | 14 +- setup.cfg | 18 + transcoding/backend_extra.py | 19 +- transcoding/tasks_extra.py | 36 +- transcoding/transcode.py | 48 ++- videofront/__init__.py | 2 +- videofront/celery_videofront.py | 13 +- videofront/settings.py | 210 +++++------ videofront/settings_prod_sample_aws.py | 38 +- videofront/urls.py | 11 +- videofront/wsgi.py | 1 + 63 files changed, 1862 insertions(+), 1191 deletions(-) create mode 100644 setup.cfg diff --git a/README.md b/README.md index 7b79604..3ab8a60 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ Pick a backend and customize the settings file accordingly: The recommended approach is to start gunicorn and celery workers with `supervisorctl`: - $ cat /etc/supervisor/conf.d/videofront.conf + $ cat /etc/supervisor/conf.d/videofront.conf [group:videofront] programs=gunicorn,celery,celery-beat @@ -141,7 +141,7 @@ The recommended approach is to start gunicorn and celery workers with `superviso Recommended nginx configuration: - $ cat /etc/nginx/sites-enabled/videofront.vhost + $ cat /etc/nginx/sites-enabled/videofront.vhost upstream django { server 127.0.0.1:8000; } @@ -156,7 +156,7 @@ Recommended nginx configuration: # This depends on the STATIC_ROOT setting alias /home/user/videofront/static/; } - + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Host $http_host; proxy_redirect off; diff --git a/api/apps.py b/api/apps.py index d87006d..14b89a8 100644 --- a/api/apps.py +++ b/api/apps.py @@ -2,4 +2,4 @@ class ApiConfig(AppConfig): - name = 'api' + name = "api" diff --git a/api/models.py b/api/models.py index 2de768f..dec317e 100644 --- a/api/models.py +++ b/api/models.py @@ -1,6 +1,7 @@ from django.conf import settings from django.db.models.signals import post_save from django.dispatch import receiver + from rest_framework.authtoken.models import Token diff --git a/api/tests/test_models.py b/api/tests/test_models.py index afb008c..7e91c83 100644 --- a/api/tests/test_models.py +++ b/api/tests/test_models.py @@ -1,10 +1,10 @@ from django.contrib.auth.models import User from django.test import TestCase + from rest_framework.authtoken.models import Token class ApiV1ModelsTests(TestCase): - def test_tokens_are_created_for_all_users(self): - user = User.objects.create(username='testuser') + user = User.objects.create(username="testuser") self.assertEqual(1, Token.objects.filter(user=user).count()) diff --git a/api/tests/v1/base.py b/api/tests/v1/base.py index 80a9be8..5bfa7ac 100644 --- a/api/tests/v1/base.py +++ b/api/tests/v1/base.py @@ -1,9 +1,8 @@ -from django.test import TestCase from django.contrib.auth.models import User +from django.test import TestCase class BaseAuthenticatedTests(TestCase): - def setUp(self): self.user = User.objects.create(username="test", is_active=True) self.user.set_password("password") diff --git a/api/tests/v1/test_playlists.py b/api/tests/v1/test_playlists.py index 3260341..65ab79c 100644 --- a/api/tests/v1/test_playlists.py +++ b/api/tests/v1/test_playlists.py @@ -1,19 +1,23 @@ from django.urls import reverse from pipeline.tests import factories + from .base import BaseAuthenticatedTests class PlaylistTests(BaseAuthenticatedTests): - def test_list_playlists_no_result(self): - response = self.client.get(reverse('api:v1:playlist-list')) + response = self.client.get(reverse("api:v1:playlist-list")) playlists = response.json() self.assertEqual([], playlists) def test_get_playlist(self): - playlist = factories.PlaylistFactory(name="Funkadelic playlist", owner=self.user) - response = self.client.get(reverse('api:v1:playlist-detail', kwargs={'id': playlist.public_id})) + playlist = factories.PlaylistFactory( + name="Funkadelic playlist", owner=self.user + ) + response = self.client.get( + reverse("api:v1:playlist-detail", kwargs={"id": playlist.public_id}) + ) result = response.json() self.assertEqual(200, response.status_code) @@ -21,57 +25,71 @@ def test_get_playlist(self): self.assertEqual(playlist.public_id, result["id"]) def test_search_playlist_by_name(self): - factories.PlaylistFactory(name="Funkadelic", owner=self.user, public_id="funkid") - factories.PlaylistFactory(name="Rockabilly", owner=self.user, public_id="rockid") + factories.PlaylistFactory( + name="Funkadelic", owner=self.user, public_id="funkid" + ) + factories.PlaylistFactory( + name="Rockabilly", owner=self.user, public_id="rockid" + ) - response_funk = self.client.get(reverse('api:v1:playlist-list'), data={'name': 'Funk'}) + response_funk = self.client.get( + reverse("api:v1:playlist-list"), data={"name": "Funk"} + ) playlists_funk = response_funk.json() self.assertEqual(1, len(playlists_funk)) - self.assertEqual('funkid', playlists_funk[0]['id']) + self.assertEqual("funkid", playlists_funk[0]["id"]) def test_insert_video_in_playlist(self): - playlist = factories.PlaylistFactory(name="Funkadelic playlist", owner=self.user) + playlist = factories.PlaylistFactory( + name="Funkadelic playlist", owner=self.user + ) factories.VideoFactory(public_id="videoid", owner=self.user) response = self.client.post( - reverse("api:v1:playlist-add-video", kwargs={'id': playlist.public_id}), - data={"id": 'videoid'} + reverse("api:v1:playlist-add-video", kwargs={"id": playlist.public_id}), + data={"id": "videoid"}, ) self.assertEqual(204, response.status_code) self.assertEqual(1, playlist.videos.count()) def test_insert_non_existing_video_in_playlist(self): - playlist = factories.PlaylistFactory(name="Funkadelic playlist", owner=self.user) + playlist = factories.PlaylistFactory( + name="Funkadelic playlist", owner=self.user + ) response = self.client.post( - reverse("api:v1:playlist-add-video", kwargs={'id': playlist.public_id}), - data={"id": 'videoid'} + reverse("api:v1:playlist-add-video", kwargs={"id": playlist.public_id}), + data={"id": "videoid"}, ) self.assertEqual(404, response.status_code) def test_insert_video_from_different_user_in_playlist(self): - playlist = factories.PlaylistFactory(name="Funkadelic playlist", owner=self.user) + playlist = factories.PlaylistFactory( + name="Funkadelic playlist", owner=self.user + ) different_user = factories.UserFactory() factories.VideoFactory(public_id="videoid", owner=different_user) response = self.client.post( - reverse("api:v1:playlist-add-video", kwargs={'id': playlist.public_id}), - data={"id": 'videoid'} + reverse("api:v1:playlist-add-video", kwargs={"id": playlist.public_id}), + data={"id": "videoid"}, ) self.assertEqual(404, response.status_code) def test_remove_video_from_playlist(self): - playlist = factories.PlaylistFactory(name="Funkadelic playlist", owner=self.user) + playlist = factories.PlaylistFactory( + name="Funkadelic playlist", owner=self.user + ) video = factories.VideoFactory(public_id="videoid", owner=self.user) playlist.videos.add(video) response = self.client.post( - reverse("api:v1:playlist-remove-video", kwargs={'id': playlist.public_id}), - data={"id": 'videoid'} + reverse("api:v1:playlist-remove-video", kwargs={"id": playlist.public_id}), + data={"id": "videoid"}, ) self.assertEqual(204, response.status_code) diff --git a/api/tests/v1/test_subtitles.py b/api/tests/v1/test_subtitles.py index 9fab706..c2455ec 100644 --- a/api/tests/v1/test_subtitles.py +++ b/api/tests/v1/test_subtitles.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals + from io import StringIO from unittest.mock import Mock, patch @@ -6,8 +7,8 @@ from django.urls import reverse from pipeline import models -from pipeline.tests.utils import override_plugin_backend from pipeline.tests import factories +from pipeline.tests.utils import override_plugin_backend from .base import BaseAuthenticatedTests @@ -25,21 +26,16 @@ class SubtitleTests(BaseAuthenticatedTests): 00:00:03,920 --> 00:00:08,250 Also I have utf8 characters: é û ë ï 你好.""" - @override_plugin_backend( upload_subtitle=lambda *args: None, - subtitle_url=lambda *args: "http://example.com/sub.vtt" + subtitle_url=lambda *args: "http://example.com/sub.vtt", ) def test_upload_subtitle(self): video = factories.VideoFactory(public_id="videoid", owner=self.user) - url = reverse("api:v1:video-subtitles", kwargs={'id': 'videoid'}) + url = reverse("api:v1:video-subtitles", kwargs={"id": "videoid"}) subfile = StringIO(self.SRT_CONTENT) response = self.client.post( - url, - data={ - 'language': 'fr', - 'name': 'sub.srt', 'file': subfile - }, + url, data={"language": "fr", "name": "sub.srt", "file": subfile} ) self.assertEqual(201, response.status_code) @@ -51,36 +47,36 @@ def test_upload_subtitle(self): def test_upload_subtitle_invalid_language(self): factories.VideoFactory(public_id="videoid", owner=self.user) - url = reverse("api:v1:video-subtitles", kwargs={'id': 'videoid'}) + url = reverse("api:v1:video-subtitles", kwargs={"id": "videoid"}) # only country codes are accepted - response = self.client.post(url, data={'language': 'french'}) + response = self.client.post(url, data={"language": "french"}) self.assertEqual(400, response.status_code) - self.assertIn('language', response.json()) + self.assertIn("language", response.json()) self.assertEqual(0, models.Subtitle.objects.count()) def test_upload_subtitle_missing_file(self): factories.VideoFactory(public_id="videoid", owner=self.user) - url = reverse("api:v1:video-subtitles", kwargs={'id': 'videoid'}) - response = self.client.post(url, data={'language': 'fr'}) + url = reverse("api:v1:video-subtitles", kwargs={"id": "videoid"}) + response = self.client.post(url, data={"language": "fr"}) self.assertEqual(400, response.status_code) - self.assertIn('file', response.json()) + self.assertIn("file", response.json()) self.assertEqual(0, models.Subtitle.objects.count()) - @patch('django.core.handlers.base.logger')# mute request logger + @patch("django.core.handlers.base.logger") # mute request logger def test_upload_subtitle_failed_upload(self, mock_logger): factories.VideoFactory(public_id="videoid", owner=self.user) - url = reverse("api:v1:video-subtitles", kwargs={'id': 'videoid'}) + url = reverse("api:v1:video-subtitles", kwargs={"id": "videoid"}) subfile = StringIO(self.SRT_CONTENT) upload_subtitle = Mock(side_effect=ValueError) with override_plugin_backend(upload_subtitle=upload_subtitle): - self.assertRaises(ValueError, self.client.post, url, - data={ - 'language': 'fr', - 'name': 'sub.srt', 'file': subfile - }, + self.assertRaises( + ValueError, + self.client.post, + url, + data={"language": "fr", "name": "sub.srt", "file": subfile}, ) self.assertEqual(0, models.Subtitle.objects.count()) @@ -88,23 +84,28 @@ def test_upload_subtitle_failed_upload(self, mock_logger): def test_cannot_modify_subtitle(self): video = factories.VideoFactory(public_id="videoid", owner=self.user) video.subtitles.create(public_id="subid", language="fr") - url = reverse("api:v1:video-subtitles", kwargs={'id': 'videoid'}) + url = reverse("api:v1:video-subtitles", kwargs={"id": "videoid"}) subfile = StringIO(self.SRT_CONTENT) with override_plugin_backend( - upload_subtitle=lambda *args: None, - subtitle_url=lambda *args: None + upload_subtitle=lambda *args: None, subtitle_url=lambda *args: None ): - response = self.client.post(url, data={ - 'id': 'subid', - 'language': 'en', - 'name': 'sub.srt', 'file': subfile - }) + response = self.client.post( + url, + data={ + "id": "subid", + "language": "en", + "name": "sub.srt", + "file": subfile, + }, + ) # Subtitle is in fact created, not modified self.assertEqual(201, response.status_code) - self.assertEqual('fr', video.subtitles.get(public_id='subid').language) - self.assertEqual('en', video.subtitles.exclude(public_id='subid').get().language) + self.assertEqual("fr", video.subtitles.get(public_id="subid").language) + self.assertEqual( + "en", video.subtitles.exclude(public_id="subid").get().language + ) @override_settings(SUBTITLES_MAX_BYTES=139) def test_upload_subtitle_too_large(self): @@ -116,52 +117,54 @@ def test_upload_subtitle_too_large(self): ) subfile = StringIO(content) self.assertEqual(140, len(content)) - url = reverse("api:v1:video-subtitles", kwargs={'id': 'videoid'}) + url = reverse("api:v1:video-subtitles", kwargs={"id": "videoid"}) - response = self.client.post(url, data={ - 'language': 'en', - 'name': 'sub.srt', 'file': subfile - }) + response = self.client.post( + url, data={"language": "en", "name": "sub.srt", "file": subfile} + ) self.assertEqual(400, response.status_code) - self.assertIn('file', response.json()) - self.assertIn('139', response.json()['file']) + self.assertIn("file", response.json()) + self.assertIn("139", response.json()["file"]) def test_upload_subtitle_invalid_format(self): factories.VideoFactory(public_id="videoid", owner=self.user) subfile = StringIO("Some invalid content here.") response = self.client.post( - reverse("api:v1:video-subtitles", kwargs={'id': 'videoid'}), data={ - 'language': 'en', - 'name': 'sub.srt', 'file': subfile - }) + reverse("api:v1:video-subtitles", kwargs={"id": "videoid"}), + data={"language": "en", "name": "sub.srt", "file": subfile}, + ) self.assertEqual(400, response.status_code) - self.assertIn('file', response.json()) - self.assertEqual('Could not detect subtitle format', response.json()['file']) + self.assertIn("file", response.json()) + self.assertEqual("Could not detect subtitle format", response.json()["file"]) self.assertEqual(0, models.Subtitle.objects.count()) def test_get_subtitle(self): - video = factories.VideoFactory(public_id='videoid', owner=self.user) - factories.SubtitleFactory(video=video, public_id='subid', language='fr') + video = factories.VideoFactory(public_id="videoid", owner=self.user) + factories.SubtitleFactory(video=video, public_id="subid", language="fr") - with override_plugin_backend( - subtitle_url=lambda *args: "http://sub.vtt" - ): - response = self.client.get(reverse("api:v1:subtitle-detail", kwargs={'id': 'subid'})) + with override_plugin_backend(subtitle_url=lambda *args: "http://sub.vtt"): + response = self.client.get( + reverse("api:v1:subtitle-detail", kwargs={"id": "subid"}) + ) self.assertEqual(200, response.status_code) subtitle = response.json() - self.assertEqual('fr', subtitle['language']) - self.assertEqual('subid', subtitle['id']) - self.assertEqual('http://sub.vtt', subtitle['url']) + self.assertEqual("fr", subtitle["language"]) + self.assertEqual("subid", subtitle["id"]) + self.assertEqual("http://sub.vtt", subtitle["url"]) def test_delete_subtitle(self): - video = factories.VideoFactory(public_id='videoid', owner=self.user) - factories.SubtitleFactory(video=video, public_id='subid', language='fr') + video = factories.VideoFactory(public_id="videoid", owner=self.user) + factories.SubtitleFactory(video=video, public_id="subid", language="fr") mock_backend = Mock(return_value=Mock(delete_subtitle=Mock())) with override_settings(PLUGIN_BACKEND=mock_backend): - response = self.client.delete(reverse("api:v1:subtitle-detail", kwargs={'id': 'subid'})) + response = self.client.delete( + reverse("api:v1:subtitle-detail", kwargs={"id": "subid"}) + ) self.assertEqual(204, response.status_code) self.assertEqual(0, models.Subtitle.objects.count()) - mock_backend.return_value.delete_subtitle.assert_called_once_with('videoid', 'subid') + mock_backend.return_value.delete_subtitle.assert_called_once_with( + "videoid", "subid" + ) diff --git a/api/tests/v1/test_users.py b/api/tests/v1/test_users.py index 7580aa3..8f74571 100644 --- a/api/tests/v1/test_users.py +++ b/api/tests/v1/test_users.py @@ -5,26 +5,25 @@ class UsersTests(BaseAuthenticatedTests): - def setUp(self): super(UsersTests, self).setUp() self.user.is_staff = True self.user.save() def test_get_user(self): - url = reverse('api:v1:user-detail', kwargs={'username': self.user.username}) + url = reverse("api:v1:user-detail", kwargs={"username": self.user.username}) response = self.client.get(url) self.assertEqual(200, response.status_code) response_data = response.json() - self.assertNotIn('password', response_data) - self.assertEqual(self.user.username, response_data['username']) - self.assertIsNotNone(response_data['token']) - self.assertTrue(response_data['is_staff']) + self.assertNotIn("password", response_data) + self.assertEqual(self.user.username, response_data["username"]) + self.assertIsNotNone(response_data["token"]) + self.assertTrue(response_data["is_staff"]) def test_list_users(self): - url = reverse('api:v1:user-list') + url = reverse("api:v1:user-list") response = self.client.get(url) self.assertEqual(200, response.status_code) @@ -33,39 +32,31 @@ def test_list_users(self): self.assertEqual(1, len(response_data)) def test_create_user_with_password(self): - url = reverse('api:v1:user-list') - response = self.client.post(url, { - 'username': 'newuser', - 'password': '1234', - }) + url = reverse("api:v1:user-list") + response = self.client.post(url, {"username": "newuser", "password": "1234"}) self.assertEqual(201, response.status_code) - self.assertEqual(1, User.objects.filter(username='newuser').count()) - newuser = User.objects.get(username='newuser') - self.assertTrue(newuser.check_password('1234')) + self.assertEqual(1, User.objects.filter(username="newuser").count()) + newuser = User.objects.get(username="newuser") + self.assertTrue(newuser.check_password("1234")) def test_create_user_without_password(self): - url = reverse('api:v1:user-list') - response = self.client.post(url, { - 'username': 'newuser', - }) + url = reverse("api:v1:user-list") + response = self.client.post(url, {"username": "newuser"}) self.assertEqual(201, response.status_code) - self.assertEqual(1, User.objects.filter(username='newuser').count()) - newuser = User.objects.get(username='newuser') + self.assertEqual(1, User.objects.filter(username="newuser").count()) + newuser = User.objects.get(username="newuser") # Check complex password was used - self.assertFalse(newuser.check_password('1234')) - self.assertFalse(newuser.check_password('')) + self.assertFalse(newuser.check_password("1234")) + self.assertFalse(newuser.check_password("")) self.assertFalse(newuser.check_password(None)) -class UsersNonAdminTests(BaseAuthenticatedTests): +class UsersNonAdminTests(BaseAuthenticatedTests): def test_create_user_fails(self): - url = reverse('api:v1:user-list') - response = self.client.post(url, { - 'username': 'newuser', - 'password': '1234', - }) + url = reverse("api:v1:user-list") + response = self.client.post(url, {"username": "newuser", "password": "1234"}) self.assertEqual(403, response.status_code) diff --git a/api/tests/v1/test_video_upload_urls.py b/api/tests/v1/test_video_upload_urls.py index 12d617b..4ae6f30 100644 --- a/api/tests/v1/test_video_upload_urls.py +++ b/api/tests/v1/test_video_upload_urls.py @@ -5,13 +5,13 @@ from django.urls import reverse from pipeline import models -from pipeline.tests.utils import override_plugin_backend from pipeline.tests import factories +from pipeline.tests.utils import override_plugin_backend + from .base import BaseAuthenticatedTests class VideoUploadUrlTests(BaseAuthenticatedTests): - def test_create_videouploadurl(self): url = reverse("api:v1:videouploadurl-list") @@ -26,8 +26,7 @@ def test_create_videouploadurl(self): def test_create_video_upload_url_with_origin(self): response = self.client.post( - reverse("api:v1:videouploadurl-list"), - data={"origin": "example.com"} + reverse("api:v1:videouploadurl-list"), data={"origin": "example.com"} ) self.assertEqual(201, response.status_code) @@ -42,13 +41,13 @@ def test_list_videouploadurls(self): def test_used_upload_urls_are_not_listed(self): factories.VideoUploadUrlFactory( - public_video_id='unused', + public_video_id="unused", owner=self.user, expires_at=time() + 3600, was_used=False, ) factories.VideoUploadUrlFactory( - public_video_id='used', + public_video_id="used", owner=self.user, expires_at=time() + 3600, was_used=True, @@ -57,21 +56,21 @@ def test_used_upload_urls_are_not_listed(self): response = self.client.get(url) self.assertEqual(1, len(response.json())) - self.assertEqual("unused", response.json()[0]['id']) + self.assertEqual("unused", response.json()[0]["id"]) def test_create_videouploadurl_with_playlist(self): playlist = factories.PlaylistFactory(owner=self.user) - response = self.client.post(reverse("api:v1:videouploadurl-list"), { - 'playlist': playlist.public_id, - }) + response = self.client.post( + reverse("api:v1:videouploadurl-list"), {"playlist": playlist.public_id} + ) self.assertEqual(201, response.status_code) self.assertEqual(playlist, models.VideoUploadUrl.objects.get().playlist) def test_obtain_video_upload_url_with_invalid_playlist_id(self): - response = self.client.post(reverse("api:v1:videouploadurl-list"), { - 'playlist': 'dummy_id', - }) + response = self.client.post( + reverse("api:v1:videouploadurl-list"), {"playlist": "dummy_id"} + ) self.assertEqual(400, response.status_code) @@ -79,22 +78,22 @@ def test_obtain_video_upload_url_with_unauthorized_playlist_id(self): # Create factory from different owner playlist = factories.PlaylistFactory() - response = self.client.post(reverse("api:v1:videouploadurl-list"), { - 'playlist': playlist.public_id, - }) + response = self.client.post( + reverse("api:v1:videouploadurl-list"), {"playlist": playlist.public_id} + ) self.assertNotEqual(self.user, playlist.owner) self.assertEqual(400, response.status_code) def test_send_file_to_upload_url(self): - self.client.logout() # upload should work even for non logged-in clients + self.client.logout() # upload should work even for non logged-in clients video_upload_url = factories.VideoUploadUrlFactory( - public_video_id='videoid', + public_video_id="videoid", owner=self.user, expires_at=time() + 3600, origin="example.com", ) - video_file = StringIO('some video content') + video_file = StringIO("some video content") upload_video = Mock() start_transcoding = Mock(return_value=[]) @@ -106,62 +105,74 @@ def test_send_file_to_upload_url(self): create_thumbnail=create_thumbnail, ): response = self.client.post( - reverse("api:v1:video-upload", kwargs={'video_id': video_upload_url.public_video_id}), - {'name': 'video.mp4', 'file': video_file}, + reverse( + "api:v1:video-upload", + kwargs={"video_id": video_upload_url.public_video_id}, + ), + {"name": "video.mp4", "file": video_file}, ) self.assertEqual(200, response.status_code) - self.assertEqual('example.com', response['Access-Control-Allow-Origin']) + self.assertEqual("example.com", response["Access-Control-Allow-Origin"]) upload_video.assert_called_once() create_thumbnail.assert_called_once() - start_transcoding.assert_called_once_with('videoid') - self.assertEqual('videoid', response.json()['id']) + start_transcoding.assert_called_once_with("videoid") + self.assertEqual("videoid", response.json()["id"]) def test_send_empty_file_to_upload_url(self): video_upload_url = factories.VideoUploadUrlFactory( - public_video_id='videoid', + public_video_id="videoid", owner=self.user, expires_at=time() + 3600, origin="*", ) response = self.client.post( - reverse("api:v1:video-upload", kwargs={'video_id': video_upload_url.public_video_id}), - {'name': 'video.mp4', 'file': StringIO('')}, + reverse( + "api:v1:video-upload", + kwargs={"video_id": video_upload_url.public_video_id}, + ), + {"name": "video.mp4", "file": StringIO("")}, ) self.assertEqual(400, response.status_code) - self.assertEqual('*', response['Access-Control-Allow-Origin']) - self.assertIn('file', response.json()) + self.assertEqual("*", response["Access-Control-Allow-Origin"]) + self.assertIn("file", response.json()) def test_send_file_to_expired_upload_url(self): video_upload_url = factories.VideoUploadUrlFactory( - public_video_id='videoid', + public_video_id="videoid", owner=self.user, expires_at=time() - 7200, origin="*", ) - video_file = StringIO('some video content') + video_file = StringIO("some video content") response = self.client.post( - reverse("api:v1:video-upload", kwargs={'video_id': video_upload_url.public_video_id}), - {'name': 'video.mp4', 'file': video_file}, + reverse( + "api:v1:video-upload", + kwargs={"video_id": video_upload_url.public_video_id}, + ), + {"name": "video.mp4", "file": video_file}, ) self.assertEqual(404, response.status_code) - self.assertNotIn('Access-Control-Allow-Origin', response) + self.assertNotIn("Access-Control-Allow-Origin", response) def test_OPTIONS_on_upload_url(self): - self.client.logout() # upload should work even for non logged-in clients + self.client.logout() # upload should work even for non logged-in clients video_upload_url = factories.VideoUploadUrlFactory( - public_video_id='videoid', + public_video_id="videoid", owner=self.user, expires_at=time() + 3600, origin="*", ) response = self.client.options( - reverse("api:v1:video-upload", kwargs={'video_id': video_upload_url.public_video_id}), + reverse( + "api:v1:video-upload", + kwargs={"video_id": video_upload_url.public_video_id}, + ) ) self.assertEqual(200, response.status_code) - self.assertIn('Access-Control-Allow-Origin', response) - self.assertEqual('*', response['Access-Control-Allow-Origin']) + self.assertIn("Access-Control-Allow-Origin", response) + self.assertEqual("*", response["Access-Control-Allow-Origin"]) diff --git a/api/tests/v1/test_videos.py b/api/tests/v1/test_videos.py index 1b3ceca..5c031b2 100644 --- a/api/tests/v1/test_videos.py +++ b/api/tests/v1/test_videos.py @@ -9,13 +9,13 @@ from django.utils.timezone import datetime, get_current_timezone from pipeline import models -from pipeline.tests.utils import override_plugin_backend from pipeline.tests import factories +from pipeline.tests.utils import override_plugin_backend + from .base import BaseAuthenticatedTests class VideosUnauthenticatedTests(TestCase): - def test_list_videos(self): url = reverse("api:v1:video-list") response = self.client.get(url) @@ -24,7 +24,9 @@ def test_list_videos(self): # Moving the cache to memory is required to obtain an accurate count of the number of SQL queries -@override_settings(CACHES={'default': {'BACKEND': 'django.core.cache.backends.locmem.LocMemCache'}}) +@override_settings( + CACHES={"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}} +) class VideosTests(BaseAuthenticatedTests): # queries: @@ -55,58 +57,72 @@ def test_list_videos_with_different_owners(self): self.assertEqual(video1.public_id, videos[0]["id"]) def test_get_video(self): - video = factories.VideoFactory(public_id="videoid", title="Some title", owner=self.user) + video = factories.VideoFactory( + public_id="videoid", title="Some title", owner=self.user + ) video.processing_state.status = models.ProcessingState.STATUS_SUCCESS video.processing_state.save() - response = self.client.get(reverse('api:v1:video-detail', kwargs={'id': 'videoid'})) + response = self.client.get( + reverse("api:v1:video-detail", kwargs={"id": "videoid"}) + ) self.assertEqual(200, response.status_code) video = response.json() - self.assertEqual('videoid', video['id']) - self.assertEqual('Some title', video['title']) - self.assertEqual([], video['subtitles']) - self.assertEqual([], video['formats']) - self.assertEqual('success', video['processing']['status']) + self.assertEqual("videoid", video["id"]) + self.assertEqual("Some title", video["title"]) + self.assertEqual([], video["subtitles"]) + self.assertEqual([], video["formats"]) + self.assertEqual("success", video["processing"]["status"]) def test_get_video_processing_state_started_at_truncated_microseconds(self): - factories.VideoFactory(public_id='videoid', title="Some title", owner=self.user) + factories.VideoFactory(public_id="videoid", title="Some title", owner=self.user) started_at = datetime(2016, 1, 1, 12, 13, 14, 1516, get_current_timezone()) - models.ProcessingState.objects.filter(video__public_id='videoid').update(started_at=started_at) + models.ProcessingState.objects.filter(video__public_id="videoid").update( + started_at=started_at + ) - response = self.client.get(reverse('api:v1:video-detail', kwargs={'id': 'videoid'})) + response = self.client.get( + reverse("api:v1:video-detail", kwargs={"id": "videoid"}) + ) video = response.json() # Check that microseconds are truncated - self.assertEqual('2016-01-01T12:13:14Z', video['processing']['started_at']) + self.assertEqual("2016-01-01T12:13:14Z", video["processing"]["started_at"]) def test_get_not_processing_video(self): - factories.VideoFactory(public_id="videoid", title='videotitle', owner=self.user) + factories.VideoFactory(public_id="videoid", title="videotitle", owner=self.user) url = reverse("api:v1:video-list") videos = self.client.get(url).json() self.assertEqual(1, len(videos)) - self.assertEqual('videoid', videos[0]['id']) - self.assertEqual('videotitle', videos[0]['title']) - self.assertIn('processing', videos[0]) - self.assertIsNotNone(videos[0]['processing']) - self.assertEqual('pending', videos[0]['processing']['status']) + self.assertEqual("videoid", videos[0]["id"]) + self.assertEqual("videotitle", videos[0]["title"]) + self.assertIn("processing", videos[0]) + self.assertIsNotNone(videos[0]["processing"]) + self.assertEqual("pending", videos[0]["processing"]["status"]) def test_get_processing_video(self): - video = factories.VideoFactory(public_id="videoid", title='videotitle', owner=self.user) + video = factories.VideoFactory( + public_id="videoid", title="videotitle", owner=self.user + ) video.processing_state.progress = 42 video.processing_state.status = models.ProcessingState.STATUS_PROCESSING video.processing_state.save() videos = self.client.get(reverse("api:v1:video-list")).json() - self.assertEqual('processing', videos[0]['processing']['status']) - self.assertEqual(42, videos[0]['processing']['progress']) + self.assertEqual("processing", videos[0]["processing"]["status"]) + self.assertEqual(42, videos[0]["processing"]["progress"]) def test_get_failed_video(self): - video = factories.VideoFactory(public_id="videoid", title='videotitle', owner=self.user) + video = factories.VideoFactory( + public_id="videoid", title="videotitle", owner=self.user + ) video.processing_state.status = models.ProcessingState.STATUS_FAILED video.processing_state.save() - response_detail = self.client.get(reverse("api:v1:video-detail", kwargs={"id": "videoid"})) + response_detail = self.client.get( + reverse("api:v1:video-detail", kwargs={"id": "videoid"}) + ) response_list = self.client.get(reverse("api:v1:video-list")) self.assertEqual(200, response_detail.status_code) @@ -116,15 +132,21 @@ def test_get_failed_video(self): def test_get_video_with_cache(self): factories.VideoFactory(public_id="videoid", title="Some title", owner=self.user) with self.assertNumQueries(self.VIDEOS_LIST_NUM_QUERIES): - response1 = self.client.get(reverse('api:v1:video-detail', kwargs={'id': 'videoid'})) + response1 = self.client.get( + reverse("api:v1:video-detail", kwargs={"id": "videoid"}) + ) with self.assertNumQueries(self.VIDEOS_LIST_NUM_QUERIES_AUTH): - response2 = self.client.get(reverse('api:v1:video-detail', kwargs={'id': 'videoid'})) + response2 = self.client.get( + reverse("api:v1:video-detail", kwargs={"id": "videoid"}) + ) self.assertEqual(200, response1.status_code) self.assertEqual(200, response2.status_code) def test_list_failed_videos(self): - video = factories.VideoFactory(public_id="videoid", title='videotitle', owner=self.user) + video = factories.VideoFactory( + public_id="videoid", title="videotitle", owner=self.user + ) video.processing_state.status = models.ProcessingState.STATUS_FAILED video.processing_state.save() @@ -133,48 +155,43 @@ def test_list_failed_videos(self): def test_create_video_fails(self): url = reverse("api:v1:video-list") - response = self.client.post( - url, - { - "public_id": "videoid", - "title": "sometitle" - } - ) - self.assertEqual(405, response.status_code) # method not allowed + response = self.client.post(url, {"public_id": "videoid", "title": "sometitle"}) + self.assertEqual(405, response.status_code) # method not allowed @override_plugin_backend( - start_transcoding=lambda video_id: [], - iter_formats=lambda video_id: [], + start_transcoding=lambda video_id: [], iter_formats=lambda video_id: [] ) def test_get_video_that_was_just_uploaded(self): factories.VideoUploadUrlFactory( - public_video_id="videoid", - expires_at=time() + 3600, - owner=self.user + public_video_id="videoid", expires_at=time() + 3600, owner=self.user + ) + response = self.client.get( + reverse("api:v1:video-detail", kwargs={"id": "videoid"}) ) - response = self.client.get(reverse("api:v1:video-detail", kwargs={'id': 'videoid'})) self.assertEqual(200, response.status_code) def test_update_video_title(self): - factories.VideoFactory(public_id="videoid", title='videotitle', owner=self.user) + factories.VideoFactory(public_id="videoid", title="videotitle", owner=self.user) response = self.client.put( - reverse('api:v1:video-detail', kwargs={'id': 'videoid'}), - data=json.dumps({'title': 'title2'}), - content_type='application/json' + reverse("api:v1:video-detail", kwargs={"id": "videoid"}), + data=json.dumps({"title": "title2"}), + content_type="application/json", ) self.assertEqual(200, response.status_code) - self.assertEqual('title2', models.Video.objects.get().title) + self.assertEqual("title2", models.Video.objects.get().title) def test_delete_video(self): mock_delete_video = Mock() factories.VideoFactory(public_id="videoid", owner=self.user) with override_plugin_backend(delete_video=mock_delete_video): - response = self.client.delete(reverse('api:v1:video-detail', kwargs={'id': 'videoid'})) + response = self.client.delete( + reverse("api:v1:video-detail", kwargs={"id": "videoid"}) + ) self.assertEqual(204, response.status_code) self.assertEqual(0, models.Video.objects.count()) - mock_delete_video.assert_called_once_with('videoid') + mock_delete_video.assert_called_once_with("videoid") @override_plugin_backend( subtitle_url=lambda vid, sid, lang: "http://example.com/{}.vtt".format(sid) @@ -185,25 +202,30 @@ def test_get_video_with_subtitles(self): video.subtitles.create(language="en", public_id="subid2") with self.assertNumQueries(self.VIDEOS_LIST_NUM_QUERIES): - video = self.client.get(reverse("api:v1:video-detail", kwargs={'id': 'videoid'})).json() - - self.assertEqual([ - { - 'id': 'subid1', - 'language': 'fr', - 'url': 'http://example.com/subid1.vtt' - }, - { - 'id': 'subid2', - 'language': 'en', - 'url': 'http://example.com/subid2.vtt' - }, - ], video['subtitles']) - + video = self.client.get( + reverse("api:v1:video-detail", kwargs={"id": "videoid"}) + ).json() + + self.assertEqual( + [ + { + "id": "subid1", + "language": "fr", + "url": "http://example.com/subid1.vtt", + }, + { + "id": "subid2", + "language": "en", + "url": "http://example.com/subid2.vtt", + }, + ], + video["subtitles"], + ) @override_plugin_backend( - video_url=lambda video_id, format_name: - "http://example.com/{}/{}.mp4".format(video_id, format_name), + video_url=lambda video_id, format_name: "http://example.com/{}/{}.mp4".format( + video_id, format_name + ), iter_formats=lambda video_id: [], ) def test_get_video_with_formats(self): @@ -212,67 +234,86 @@ def test_get_video_with_formats(self): video.formats.create(name="HD", bitrate=256) with self.assertNumQueries(self.VIDEOS_LIST_NUM_QUERIES): - video = self.client.get(reverse("api:v1:video-detail", kwargs={'id': 'videoid'})).json() - - self.assertEqual([ - { - 'name': 'SD', - 'url': 'http://example.com/videoid/SD.mp4', - 'bitrate': 128 - }, - { - 'name': 'HD', - 'url': 'http://example.com/videoid/HD.mp4', - 'bitrate': 256 - }, - ], video['formats']) + video = self.client.get( + reverse("api:v1:video-detail", kwargs={"id": "videoid"}) + ).json() + + self.assertEqual( + [ + { + "name": "SD", + "url": "http://example.com/videoid/SD.mp4", + "bitrate": 128, + }, + { + "name": "HD", + "url": "http://example.com/videoid/HD.mp4", + "bitrate": 256, + }, + ], + video["formats"], + ) def test_list_videos_in_playlist(self): - playlist = factories.PlaylistFactory(name="Funkadelic playlist", owner=self.user) + playlist = factories.PlaylistFactory( + name="Funkadelic playlist", owner=self.user + ) video_in_playlist = factories.VideoFactory(owner=self.user) _video_not_in_playlist = factories.VideoFactory(owner=self.user) playlist.videos.add(video_in_playlist) - response = self.client.get(reverse('api:v1:video-list'), data={'playlist_id': playlist.public_id}) + response = self.client.get( + reverse("api:v1:video-list"), data={"playlist_id": playlist.public_id} + ) videos = response.json() self.assertEqual(1, len(videos)) self.assertEqual(video_in_playlist.public_id, videos[0]["id"]) @override_plugin_backend( - thumbnail_url=lambda video_id, thumb_id: "http://imgur.com/{}/thumbs/{}.jpg".format(video_id, thumb_id), + thumbnail_url=lambda video_id, thumb_id: "http://imgur.com/{}/thumbs/{}.jpg".format( + video_id, thumb_id + ) ) def test_get_video_thumbnail(self): - factories.VideoFactory(public_id="videoid", owner=self.user, public_thumbnail_id="thumbid") - video = self.client.get(reverse("api:v1:video-detail", kwargs={'id': 'videoid'})).json() - self.assertIn('thumbnail', video) - self.assertEqual("http://imgur.com/videoid/thumbs/thumbid.jpg", video['thumbnail']) + factories.VideoFactory( + public_id="videoid", owner=self.user, public_thumbnail_id="thumbid" + ) + video = self.client.get( + reverse("api:v1:video-detail", kwargs={"id": "videoid"}) + ).json() + self.assertIn("thumbnail", video) + self.assertEqual( + "http://imgur.com/videoid/thumbs/thumbid.jpg", video["thumbnail"] + ) - @patch('pipeline.utils.resize_image') + @patch("pipeline.utils.resize_image") @override_plugin_backend( upload_thumbnail=lambda video_id, thumb_id, file_object: None, delete_thumbnail=lambda video_id, thumb_id: None, - thumbnail_url=lambda video_id, thumb_id: "http://example.com/{}/{}.jpg".format(video_id, thumb_id) + thumbnail_url=lambda video_id, thumb_id: "http://example.com/{}/{}.jpg".format( + video_id, thumb_id + ), ) def test_upload_video_thumbnail(self, mock_resize_image): video = factories.VideoFactory(public_id="videoid", owner=self.user) old_thumbnail_url = video.thumbnail_url - url = reverse("api:v1:video-thumbnail", kwargs={'id': 'videoid'}) + url = reverse("api:v1:video-thumbnail", kwargs={"id": "videoid"}) thumb_file = BytesIO(b"thumb content") thumb_file.name = "thumb.jpg" response = self.client.post(url, {"name": "thumb.jpg", "file": thumb_file}) self.assertEqual(200, response.status_code) - self.assertIn('thumbnail', response.json()) - self.assertIn("http://example.com/videoid", response.json()['thumbnail']) - self.assertNotEqual(old_thumbnail_url, response.json()['thumbnail']) + self.assertIn("thumbnail", response.json()) + self.assertIn("http://example.com/videoid", response.json()["thumbnail"]) + self.assertNotEqual(old_thumbnail_url, response.json()["thumbnail"]) def test_upload_invalid_video_thumbnail(self): factories.VideoFactory(public_id="videoid", owner=self.user) - url = reverse("api:v1:video-thumbnail", kwargs={'id': 'videoid'}) + url = reverse("api:v1:video-thumbnail", kwargs={"id": "videoid"}) thumb_file = BytesIO(b"invalid thumb content") thumb_file.name = "thumb.jpg" response = self.client.post(url, {"name": "thumb.jpg", "file": thumb_file}) self.assertEqual(400, response.status_code) - self.assertIn('file', response.json()) + self.assertIn("file", response.json()) diff --git a/api/tests/v1/test_views.py b/api/tests/v1/test_views.py index 296ccf3..c9bff6c 100644 --- a/api/tests/v1/test_views.py +++ b/api/tests/v1/test_views.py @@ -4,7 +4,6 @@ class ApiV1Tests(TestCase): - def test_unauthenticated_root(self): url = reverse("api:v1:api-root") response = self.client.get(url) @@ -14,21 +13,16 @@ def test_home_redirects_to_api(self): url = reverse("home") response = self.client.get(url, follow=True) self.assertEqual(200, response.status_code) - self.assertEqual([(reverse('api:v1:api-root'), 302)], response.redirect_chain) + self.assertEqual([(reverse("api:v1:api-root"), 302)], response.redirect_chain) def test_get_token(self): - user = User.objects.create(username='testuser') - user.set_password('password') + user = User.objects.create(username="testuser") + user.set_password("password") user.save() url = reverse("api:v1:auth-token") - response = self.client.post(url, { - 'username': user.username, - 'password': 'password', - }) - self.assertEqual(200, response.status_code) - self.assertEqual( - {'token': user.auth_token.key}, - response.json() + response = self.client.post( + url, {"username": user.username, "password": "password"} ) - + self.assertEqual(200, response.status_code) + self.assertEqual({"token": user.auth_token.key}, response.json()) diff --git a/api/urls.py b/api/urls.py index 1657f67..e33ec38 100644 --- a/api/urls.py +++ b/api/urls.py @@ -1,8 +1,6 @@ -from django.conf.urls import url, include +from django.conf.urls import include, url app_name = "api" -urlpatterns = [ - url(r'^v1/', include('api.v1.urls')), -] +urlpatterns = [url(r"^v1/", include("api.v1.urls"))] diff --git a/api/v1/serializers.py b/api/v1/serializers.py index 9481558..c08e67d 100644 --- a/api/v1/serializers.py +++ b/api/v1/serializers.py @@ -1,18 +1,20 @@ from time import time from django.contrib.auth.models import User + from rest_framework import serializers from pipeline import models + from . import utils class PlaylistSerializer(serializers.ModelSerializer): - id = serializers.CharField(source='public_id', read_only=True) + id = serializers.CharField(source="public_id", read_only=True) owner = serializers.HiddenField(default=serializers.CurrentUserDefault()) class Meta: - fields = ('id', 'name', 'owner') + fields = ("id", "name", "owner") model = models.Playlist @@ -20,52 +22,51 @@ class ProcessingStateSerializer(serializers.ModelSerializer): started_at = serializers.DateTimeField(format="%Y-%m-%dT%H:%M:%SZ") class Meta: - fields = ('status', 'progress', 'started_at') + fields = ("status", "progress", "started_at") model = models.ProcessingState class SubtitleSerializer(serializers.ModelSerializer): - id = serializers.CharField(source='public_id', read_only=True) - video_id = serializers.CharField(source='video__id', read_only=True) + id = serializers.CharField(source="public_id", read_only=True) + video_id = serializers.CharField(source="video__id", read_only=True) url = serializers.CharField(read_only=True) class Meta: - fields = ('id', 'language', 'video_id', 'url') + fields = ("id", "language", "video_id", "url") model = models.Subtitle class VideoUploadUrlSerializer(serializers.ModelSerializer): class RelatedPlaylistField(serializers.SlugRelatedField): def get_queryset(self): - return models.Playlist.objects.filter(owner=self.context['request'].user) + return models.Playlist.objects.filter(owner=self.context["request"].user) - id = serializers.CharField(source='public_video_id', read_only=True) + id = serializers.CharField(source="public_video_id", read_only=True) expires_at = serializers.IntegerField( default=serializers.CreateOnlyDefault( lambda: time() + models.VideoUploadUrl.objects.EXPIRE_DELAY ) ) owner = serializers.HiddenField(default=serializers.CurrentUserDefault()) - playlist = RelatedPlaylistField(slug_field='public_id', required=False) + playlist = RelatedPlaylistField(slug_field="public_id", required=False) class Meta: - fields = ('id', 'expires_at', 'owner', 'origin', 'playlist',) + fields = ("id", "expires_at", "owner", "origin", "playlist") model = models.VideoUploadUrl class UserSerializer(serializers.ModelSerializer): password = serializers.CharField(write_only=True, default=utils.random_password) - token = serializers.CharField(read_only=True, source='auth_token.key') + token = serializers.CharField(read_only=True, source="auth_token.key") is_staff = serializers.BooleanField(read_only=True) class Meta: - fields = ('username', 'password', 'token', 'is_staff',) + fields = ("username", "password", "token", "is_staff") model = User def create(self, validated_data): return User.objects.create_user( - validated_data['username'], - password=validated_data['password'] + validated_data["username"], password=validated_data["password"] ) @@ -74,17 +75,17 @@ class VideoFormatSerializer(serializers.ModelSerializer): bitrate = serializers.FloatField(read_only=True) class Meta: - fields = ('name', 'url', 'bitrate',) + fields = ("name", "url", "bitrate") model = models.VideoFormat class VideoSerializer(serializers.ModelSerializer): - id = serializers.CharField(source='public_id', read_only=True) - processing = ProcessingStateSerializer(source='processing_state', read_only=True) + id = serializers.CharField(source="public_id", read_only=True) + processing = ProcessingStateSerializer(source="processing_state", read_only=True) subtitles = SubtitleSerializer(many=True, read_only=True) formats = VideoFormatSerializer(many=True, read_only=True) - thumbnail = serializers.CharField(source='thumbnail_url', read_only=True) + thumbnail = serializers.CharField(source="thumbnail_url", read_only=True) class Meta: - fields = ('id', 'title', 'processing', 'subtitles', 'formats', 'thumbnail',) + fields = ("id", "title", "processing", "subtitles", "formats", "thumbnail") model = models.Video diff --git a/api/v1/urls.py b/api/v1/urls.py index 490a235..3532a95 100644 --- a/api/v1/urls.py +++ b/api/v1/urls.py @@ -1,7 +1,7 @@ -from django.conf.urls import url, include +from django.conf.urls import include, url -from rest_framework.authtoken import views as authtoken_views from rest_framework import routers +from rest_framework.authtoken import views as authtoken_views from . import views @@ -13,6 +13,7 @@ class Router(routers.DefaultRouter): """ We override the router in order to provide some documentation to the API root. """ + def get_api_root_view(self, api_urls=None): root_view = super(Router, self).get_api_root_view(api_urls=api_urls) root_view.cls.__doc__ = """List of all the endpoints from the videofront API. @@ -21,17 +22,20 @@ def get_api_root_view(self, api_urls=None): """ return root_view + router = Router() -router.register(r'playlists', views.PlaylistViewSet, base_name='playlist') -router.register(r'subtitles', views.SubtitleViewSet, base_name='subtitle') -router.register(r'users', views.UserViewSet) -router.register(r'videos', views.VideoListViewSet, base_name='video') -router.register(r'videos', views.VideoViewSet, base_name='video') -router.register(r'videos', views.UploadViewset, base_name='video') -router.register(r'videouploadurls', views.VideoUploadUrlViewSet, base_name='videouploadurl') +router.register(r"playlists", views.PlaylistViewSet, base_name="playlist") +router.register(r"subtitles", views.SubtitleViewSet, base_name="subtitle") +router.register(r"users", views.UserViewSet) +router.register(r"videos", views.VideoListViewSet, base_name="video") +router.register(r"videos", views.VideoViewSet, base_name="video") +router.register(r"videos", views.UploadViewset, base_name="video") +router.register( + r"videouploadurls", views.VideoUploadUrlViewSet, base_name="videouploadurl" +) urlpatterns = [ - url(r'^', include(router.urls)), - url(r'^docs$', views.schema_view), - url(r'^auth-token/', authtoken_views.obtain_auth_token, name='auth-token') + url(r"^", include(router.urls)), + url(r"^docs$", views.schema_view), + url(r"^auth-token/", authtoken_views.obtain_auth_token, name="auth-token"), ] diff --git a/api/v1/utils.py b/api/v1/utils.py index 2aa0697..0a5c8c5 100644 --- a/api/v1/utils.py +++ b/api/v1/utils.py @@ -6,4 +6,4 @@ def random_password(length=20): """ Return a random password of given length. """ - return ''.join([random.choice(string.printable) for _ in range(0, length)]) + return "".join([random.choice(string.printable) for _ in range(0, length)]) diff --git a/api/v1/views.py b/api/v1/views.py index 429083c..b364eef 100644 --- a/api/v1/views.py +++ b/api/v1/views.py @@ -2,26 +2,32 @@ from django.conf import settings from django.contrib.auth.models import User from django.db import transaction -import django_filters -from django_filters import rest_framework as filters -from rest_framework import mixins -from rest_framework import status as rest_status -from rest_framework import viewsets -from rest_framework.authentication import SessionAuthentication, BasicAuthentication, TokenAuthentication + +from rest_framework import mixins, status as rest_status, viewsets +from rest_framework.authentication import ( + BasicAuthentication, + SessionAuthentication, + TokenAuthentication, +) from rest_framework.decorators import action, api_view, renderer_classes -from rest_framework.permissions import IsAuthenticated, IsAdminUser +from rest_framework.permissions import IsAdminUser, IsAuthenticated from rest_framework.response import Response from rest_framework.schemas import SchemaGenerator from rest_framework_swagger.renderers import OpenAPIRenderer, SwaggerUIRenderer -from pipeline import cache -from pipeline import exceptions -from pipeline import models -from pipeline import tasks +import django_filters +from django_filters import rest_framework as filters + +from pipeline import cache, exceptions, models, tasks + from . import serializers -AUTHENTICATION_CLASSES = (BasicAuthentication, SessionAuthentication, TokenAuthentication) +AUTHENTICATION_CLASSES = ( + BasicAuthentication, + SessionAuthentication, + TokenAuthentication, +) PERMISSION_CLASSES = (IsAuthenticated,) @@ -31,7 +37,7 @@ def schema_view(request): """ Swagger API documentation """ - generator = SchemaGenerator(title='Videofront API') + generator = SchemaGenerator(title="Videofront API") return Response(generator.get_schema(request=request)) @@ -39,24 +45,26 @@ class PlaylistFilter(filters.FilterSet): """ Filter playlists by name. """ + name = django_filters.CharFilter(lookup_expr="icontains") class Meta: model = models.Playlist - fields = ['name'] + fields = ["name"] class PlaylistViewSet(viewsets.ModelViewSet): """ List, update and create video playlists. """ + authentication_classes = AUTHENTICATION_CLASSES permission_classes = PERMISSION_CLASSES serializer_class = serializers.PlaylistSerializer - lookup_field = 'public_id' - lookup_url_kwarg = 'id' + lookup_field = "public_id" + lookup_url_kwarg = "id" filter_backends = (filters.DjangoFilterBackend,) filter_class = PlaylistFilter @@ -64,8 +72,7 @@ class PlaylistViewSet(viewsets.ModelViewSet): def get_queryset(self): return models.Playlist.objects.filter(owner=self.request.user) - - @action(methods=['post'], detail=True) + @action(methods=["post"], detail=True) def add_video(self, request, **kwargs): """ Add a video to a playlist @@ -79,7 +86,7 @@ def add_video(self, request, **kwargs): playlist.videos.add(video) return Response(status=rest_status.HTTP_204_NO_CONTENT) - @action(methods=['post'], detail=True) + @action(methods=["post"], detail=True) def remove_video(self, request, **kwargs): """ Remove a video from a playlist @@ -111,39 +118,42 @@ def _get_playlist_video(self, request, **kwargs): public_video_id = request.data.get("id") if not public_video_id: - raise ErrorResponse({'id': "Missing argument"}, status=rest_status.HTTP_400_BAD_REQUEST) + raise ErrorResponse( + {"id": "Missing argument"}, status=rest_status.HTTP_400_BAD_REQUEST + ) try: - video = models.Video.objects.filter( - owner=request.user - ).exclude( - processing_state__status=models.ProcessingState.STATUS_FAILED - ).get(public_id=public_video_id) + video = ( + models.Video.objects.filter(owner=request.user) + .exclude(processing_state__status=models.ProcessingState.STATUS_FAILED) + .get(public_id=public_video_id) + ) except models.Video.DoesNotExist: - raise ErrorResponse({'id': "Video does not exist"}, status=rest_status.HTTP_404_NOT_FOUND) + raise ErrorResponse( + {"id": "Video does not exist"}, status=rest_status.HTTP_404_NOT_FOUND + ) return playlist, video -class SubtitleViewSet(mixins.RetrieveModelMixin, - mixins.DestroyModelMixin, - viewsets.GenericViewSet): +class SubtitleViewSet( + mixins.RetrieveModelMixin, mixins.DestroyModelMixin, viewsets.GenericViewSet +): authentication_classes = AUTHENTICATION_CLASSES permission_classes = PERMISSION_CLASSES serializer_class = serializers.SubtitleSerializer - lookup_field = 'public_id' - lookup_url_kwarg = 'id' - + lookup_field = "public_id" + lookup_url_kwarg = "id" def get_queryset(self): - queryset = models.Subtitle.objects.select_related( - 'video' - ).exclude( - video__processing_state__status=models.ProcessingState.STATUS_FAILED - ).filter( - video__owner=self.request.user + queryset = ( + models.Subtitle.objects.select_related("video") + .exclude( + video__processing_state__status=models.ProcessingState.STATUS_FAILED + ) + .filter(video__owner=self.request.user) ) return queryset @@ -152,10 +162,12 @@ def perform_destroy(self, instance): tasks.delete_subtitle(instance.video.public_id, instance.public_id) -class UserViewSet(mixins.RetrieveModelMixin, - mixins.CreateModelMixin, - mixins.ListModelMixin, - viewsets.GenericViewSet): +class UserViewSet( + mixins.RetrieveModelMixin, + mixins.CreateModelMixin, + mixins.ListModelMixin, + viewsets.GenericViewSet, +): """ User creation, listing and details. Note that this viewset is only accessible to admin (staff) users. @@ -164,11 +176,11 @@ class UserViewSet(mixins.RetrieveModelMixin, authentication_classes = AUTHENTICATION_CLASSES permission_classes = (IsAuthenticated, IsAdminUser) - queryset = User.objects.all().order_by('-date_joined').select_related('auth_token') + queryset = User.objects.all().order_by("-date_joined").select_related("auth_token") serializer_class = serializers.UserSerializer - lookup_field = 'username' - lookup_url_kwarg = 'username' + lookup_field = "username" + lookup_url_kwarg = "username" class Meta: model = User @@ -178,35 +190,36 @@ class VideoFilter(filters.FilterSet): """ Filter videos by playlist public id. """ - playlist_id = django_filters.CharFilter(field_name="playlists", lookup_expr="public_id") + + playlist_id = django_filters.CharFilter( + field_name="playlists", lookup_expr="public_id" + ) class Meta: model = models.Video - fields = ['playlist_id'] + fields = ["playlist_id"] class VideoQuerysetMixin(object): - def get_queryset(self): # Note that here we do not exclude failed videos - queryset = models.Video.objects.select_related( - 'processing_state' - ).prefetch_related( - 'subtitles', 'formats' - ).filter( - owner=self.request.user + queryset = ( + models.Video.objects.select_related("processing_state") + .prefetch_related("subtitles", "formats") + .filter(owner=self.request.user) ) return queryset -class VideoListViewSet(mixins.ListModelMixin, - VideoQuerysetMixin, - viewsets.GenericViewSet): +class VideoListViewSet( + mixins.ListModelMixin, VideoQuerysetMixin, viewsets.GenericViewSet +): """ List available videos. Note that you may obtain only the videos that belong to a certain playlist by passing the argument `?playlist_id=xxxx`. """ + # Similar to a generic model viewset, but without creation features. Video # creation is only available through upload. @@ -218,22 +231,26 @@ class VideoListViewSet(mixins.ListModelMixin, filter_backends = (filters.DjangoFilterBackend,) filter_class = VideoFilter - def get_queryset(self): - return super(VideoListViewSet, self).get_queryset().exclude( - processing_state__status=models.ProcessingState.STATUS_FAILED + return ( + super(VideoListViewSet, self) + .get_queryset() + .exclude(processing_state__status=models.ProcessingState.STATUS_FAILED) ) -class VideoViewSet(mixins.RetrieveModelMixin, - mixins.UpdateModelMixin, - mixins.DestroyModelMixin, - VideoQuerysetMixin, - viewsets.GenericViewSet): +class VideoViewSet( + mixins.RetrieveModelMixin, + mixins.UpdateModelMixin, + mixins.DestroyModelMixin, + VideoQuerysetMixin, + viewsets.GenericViewSet, +): """ Viewset for individual videos. This is a view that allows a user to access videos that have failed transcoding. """ + # Similar to a generic model viewset, but without creation features. Video # creation is only available through upload. @@ -242,9 +259,8 @@ class VideoViewSet(mixins.RetrieveModelMixin, serializer_class = serializers.VideoSerializer - lookup_field = 'public_id' - lookup_url_kwarg = 'id' - + lookup_field = "public_id" + lookup_url_kwarg = "id" def retrieve(self, request, *args, **kwargs): # We override the `retrieve` method in order to cache API results for @@ -263,7 +279,7 @@ def perform_destroy(self, instance): super(VideoViewSet, self).perform_destroy(instance) tasks.delete_video(instance.public_id) - @action(methods=['post'], detail=True) + @action(methods=["post"], detail=True) def subtitles(self, request, **kwargs): """ Subtitle upload @@ -277,15 +293,17 @@ def subtitles(self, request, **kwargs): attachment = request.FILES.get("file") if not attachment: - return Response({'file': "Missing file"}, status=rest_status.HTTP_400_BAD_REQUEST) + return Response( + {"file": "Missing file"}, status=rest_status.HTTP_400_BAD_REQUEST + ) if attachment.size > settings.SUBTITLES_MAX_BYTES: return Response( { - 'file': "File too large. Maximum allowed size: {} bytes".format( + "file": "File too large. Maximum allowed size: {} bytes".format( settings.SUBTITLES_MAX_BYTES ) }, - status=rest_status.HTTP_400_BAD_REQUEST + status=rest_status.HTTP_400_BAD_REQUEST, ) try: @@ -293,13 +311,20 @@ def subtitles(self, request, **kwargs): # case of upload failure with transaction.atomic(): subtitle = serializer.save(video_id=video.id) - tasks.upload_subtitle(video.public_id, subtitle.public_id, subtitle.language, attachment.read()) + tasks.upload_subtitle( + video.public_id, + subtitle.public_id, + subtitle.language, + attachment.read(), + ) except exceptions.SubtitleInvalid as e: - return Response({'file': e.args[0]}, status=rest_status.HTTP_400_BAD_REQUEST) + return Response( + {"file": e.args[0]}, status=rest_status.HTTP_400_BAD_REQUEST + ) return Response(serializer.data, status=rest_status.HTTP_201_CREATED) - @action(methods=['post'], detail=True) + @action(methods=["post"], detail=True) def thumbnail(self, request, **kwargs): """ Thumbnail upload @@ -313,16 +338,20 @@ def thumbnail(self, request, **kwargs): attachment = request.FILES.get("file") if not attachment: - return Response({'file': "Missing file"}, status=rest_status.HTTP_400_BAD_REQUEST) + return Response( + {"file": "Missing file"}, status=rest_status.HTTP_400_BAD_REQUEST + ) try: tasks.upload_thumbnail(video.public_id, attachment) except exceptions.ThumbnailInvalid: - return Response({'file': "Invalid image"}, status=rest_status.HTTP_400_BAD_REQUEST) + return Response( + {"file": "Invalid image"}, status=rest_status.HTTP_400_BAD_REQUEST + ) return Response( - {'thumbnail': models.Video.objects.get(pk=video.pk).thumbnail_url}, - status=rest_status.HTTP_200_OK + {"thumbnail": models.Video.objects.get(pk=video.pk).thumbnail_url}, + status=rest_status.HTTP_200_OK, ) @@ -332,29 +361,32 @@ class VideoUploadUrlViewSet(viewsets.ModelViewSet): any user (even unauthenticated users) to upload a new video. Once a video has been uploaded, the corresponding video upload url is marked as used. """ + authentication_classes = AUTHENTICATION_CLASSES permission_classes = PERMISSION_CLASSES serializer_class = serializers.VideoUploadUrlSerializer - lookup_field = 'public_video_id' - lookup_url_kwarg = 'id' + lookup_field = "public_video_id" + lookup_url_kwarg = "id" def get_queryset(self): return models.VideoUploadUrl.objects.available().filter(owner=self.request.user) class UploadViewset(viewsets.ViewSet): - lookup_field = 'public_video_id' - lookup_url_kwarg = 'video_id' + lookup_field = "public_video_id" + lookup_url_kwarg = "video_id" - @action(methods=['post', 'options'], detail=True) + @action(methods=["post", "options"], detail=True) def upload(self, request, video_id=None): """ Upload a video file. """ try: - video_upload_url = models.VideoUploadUrl.objects.available().get(public_video_id=video_id) + video_upload_url = models.VideoUploadUrl.objects.available().get( + public_video_id=video_id + ) except models.VideoUploadUrl.DoesNotExist: return Response(status=rest_status.HTTP_404_NOT_FOUND) @@ -364,15 +396,19 @@ def upload(self, request, video_id=None): cors_headers["Access-Control-Allow-Origin"] = video_upload_url.origin # OPTIONS call - if request.method == 'OPTIONS': + if request.method == "OPTIONS": return Response({}, headers=cors_headers) # POST call - video_file = request.FILES.get('file') + video_file = request.FILES.get("file") if video_file is None or video_file.size == 0: - return Response({'file': "Missing argument"}, status=rest_status.HTTP_400_BAD_REQUEST, headers=cors_headers) + return Response( + {"file": "Missing argument"}, + status=rest_status.HTTP_400_BAD_REQUEST, + headers=cors_headers, + ) tasks.upload_video(video_upload_url.public_video_id, video_file) - return Response({'id': video_upload_url.public_video_id}, headers=cors_headers) + return Response({"id": video_upload_url.public_video_id}, headers=cors_headers) class ErrorResponse(Exception): diff --git a/contrib/plugins/aws/apps.py b/contrib/plugins/aws/apps.py index c0d9eb9..f190d7f 100644 --- a/contrib/plugins/aws/apps.py +++ b/contrib/plugins/aws/apps.py @@ -2,4 +2,4 @@ class ContribPluginsAwsConfig(AppConfig): - name = 'contrib.plugins.aws' + name = "contrib.plugins.aws" diff --git a/contrib/plugins/aws/backend.py b/contrib/plugins/aws/backend.py index 91f858e..986c042 100644 --- a/contrib/plugins/aws/backend.py +++ b/contrib/plugins/aws/backend.py @@ -1,7 +1,8 @@ from tempfile import NamedTemporaryFile -from botocore.exceptions import ClientError import boto3 +from botocore.exceptions import ClientError + from django.conf import settings import pipeline.backend @@ -13,7 +14,7 @@ class Backend(pipeline.backend.BaseBackend): VIDEO_FOLDER_KEY_PATTERN = "videos/{video_id}/" VIDEO_KEY_PATTERN = VIDEO_FOLDER_KEY_PATTERN + "{resolution}.mp4" SUBTITLE_BASE_KEY_PATTERN = VIDEO_FOLDER_KEY_PATTERN + "subs/{subtitle_id}." - SUBTITLE_KEY_PATTERN = SUBTITLE_BASE_KEY_PATTERN + "{language}.vtt" + SUBTITLE_KEY_PATTERN = SUBTITLE_BASE_KEY_PATTERN + "{language}.vtt" def __init__(self): self._session = None @@ -28,20 +29,22 @@ def session(self): if self._session is None: self._session = boto3.Session( aws_access_key_id=settings.AWS_ACCESS_KEY_ID, - aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, ) return self._session @property def s3_client(self): if self._s3_client is None: - self._s3_client = self.session.client('s3', region_name=settings.AWS_REGION) + self._s3_client = self.session.client("s3", region_name=settings.AWS_REGION) return self._s3_client @property def elastictranscoder_client(self): if self._elastictranscoder_client is None: - self._elastictranscoder_client = self.session.client('elastictranscoder', region_name=settings.AWS_REGION) + self._elastictranscoder_client = self.session.client( + "elastictranscoder", region_name=settings.AWS_REGION + ) return self._elastictranscoder_client @classmethod @@ -60,11 +63,13 @@ def get_video_key(cls, video_id, resolution): @classmethod def get_subtitle_key(cls, video_id, subtitle_id, language): - return cls.SUBTITLE_KEY_PATTERN.format(video_id=video_id, subtitle_id=subtitle_id, language=language) + return cls.SUBTITLE_KEY_PATTERN.format( + video_id=video_id, subtitle_id=subtitle_id, language=language + ) @classmethod - def get_thumbnail_key(cls, video_id, thumb_id, ext='jpg'): - return cls.get_video_folder_key(video_id) + 'thumbs/{}.{}'.format(thumb_id, ext) + def get_thumbnail_key(cls, video_id, thumb_id, ext="jpg"): + return cls.get_video_folder_key(video_id) + "thumbs/{}.{}".format(thumb_id, ext) def get_src_file_key(self, public_video_id): """ @@ -75,21 +80,20 @@ def get_src_file_key(self, public_video_id): Returns None if no source file exists. """ bucket = settings.S3_PRIVATE_BUCKET - src_folder_key = self.get_video_folder_key(public_video_id) + 'src/' + src_folder_key = self.get_video_folder_key(public_video_id) + "src/" objects = self.s3_client.list_objects(Bucket=bucket, Prefix=src_folder_key) - if objects.get('Contents'): - return objects['Contents'][0]['Key'] + if objects.get("Contents"): + return objects["Contents"][0]["Key"] return None def _get_download_base_url(self): - cloudfront = getattr(settings, 'CLOUDFRONT_DOMAIN_NAME', None) + cloudfront = getattr(settings, "CLOUDFRONT_DOMAIN_NAME", None) if cloudfront: # Download from cloudfront return "https://{domain}".format(domain=cloudfront) else: return "https://s3-{region}.amazonaws.com/{bucket}".format( - region=settings.AWS_REGION, - bucket=settings.S3_BUCKET, + region=settings.AWS_REGION, bucket=settings.S3_BUCKET ) def _get_default_acl(self): @@ -97,10 +101,10 @@ def _get_default_acl(self): If we are using a CDN, then objects are stored with private ACL by default. Else, with public-read ACL. """ - if hasattr(settings, 'CLOUDFRONT_DOMAIN_NAME'): - return 'private' + if hasattr(settings, "CLOUDFRONT_DOMAIN_NAME"): + return "private" else: - return 'public-read' + return "public-read" #################### # Overridden methods @@ -111,12 +115,12 @@ def upload_video(self, public_video_id, file_object): Store a video file on S3. """ # Source videos do not need to be accessible - acl = 'private' + acl = "private" self.s3_client.put_object( ACL=acl, Body=file_object, Bucket=settings.S3_PRIVATE_BUCKET, - Key=self.get_video_folder_key(public_video_id) + 'src/' + file_object.name, + Key=self.get_video_folder_key(public_video_id) + "src/" + file_object.name, ) def start_transcoding(self, public_video_id): @@ -129,42 +133,44 @@ def start_transcoding(self, public_video_id): output = { # Note that the transcoded video should have public-read # permissions or be accessible by cloudfront - 'Key': self.get_video_key(public_video_id, resolution), - 'PresetId': preset_id + "Key": self.get_video_key(public_video_id, resolution), + "PresetId": preset_id, } # Generate thumbnails if preset_id == settings.ELASTIC_TRANSCODER_THUMBNAILS_PRESET: - output['ThumbnailPattern'] = self.get_video_folder_key(public_video_id) + 'thumbs/{count}' + output["ThumbnailPattern"] = ( + self.get_video_folder_key(public_video_id) + "thumbs/{count}" + ) job = self.elastictranscoder_client.create_job( - PipelineId=pipeline_id, - Input={'Key': src_file_key}, - Output=output + PipelineId=pipeline_id, Input={"Key": src_file_key}, Output=output ) - jobs.append(job['Job']) + jobs.append(job["Job"]) return jobs def check_progress(self, job): - job_id = job['Id'] + job_id = job["Id"] job_update = self.elastictranscoder_client.read_job(Id=job_id) - job_status = job_update['Job']['Output']['Status'] - if job_status == 'Submitted' or job_status == 'Progressing': + job_status = job_update["Job"]["Output"]["Status"] + if job_status == "Submitted" or job_status == "Progressing": # Elastic Transcoder does not provide any indicator of the time left return 0, False - elif job_status == 'Complete': + elif job_status == "Complete": return 100, True - elif job_status == 'Error': - error_message = job_update['Job']['Output']['StatusDetail'] + elif job_status == "Error": + error_message = job_update["Job"]["Output"]["StatusDetail"] raise TranscodingFailed(error_message) else: - raise TranscodingFailed('Unknown transcoding status: {}'.format(job_status)) + raise TranscodingFailed("Unknown transcoding status: {}".format(job_status)) def delete_video(self, public_video_id): folder = self.get_video_folder_key(public_video_id) self.delete_objects(folder) def delete_subtitle(self, public_video_id, public_subtitle_id): - prefix = self.SUBTITLE_BASE_KEY_PATTERN.format(video_id=public_video_id, subtitle_id=public_subtitle_id) + prefix = self.SUBTITLE_BASE_KEY_PATTERN.format( + video_id=public_video_id, subtitle_id=public_subtitle_id + ) self.delete_objects(prefix) def delete_objects(self, prefix): @@ -175,18 +181,15 @@ def delete_objects(self, prefix): """ for bucket in [settings.S3_BUCKET, settings.S3_PRIVATE_BUCKET]: list_objects = self.s3_client.list_objects(Bucket=bucket, Prefix=prefix) - for obj in list_objects.get('Contents', []): - self.s3_client.delete_object( - Bucket=bucket, - Key=obj['Key'] - ) + for obj in list_objects.get("Contents", []): + self.s3_client.delete_object(Bucket=bucket, Key=obj["Key"]) def iter_formats(self, public_video_id): for resolution, _preset_id, bitrate in settings.ELASTIC_TRANSCODER_PRESETS: try: self.s3_client.head_object( Bucket=settings.S3_BUCKET, - Key=self.get_video_key(public_video_id, resolution) + Key=self.get_video_key(public_video_id, resolution), ) except ClientError: continue @@ -204,13 +207,13 @@ def create_thumbnail(self, video_id, thumb_id): # Download 00001.png response = self.s3_client.get_object( Bucket=settings.S3_BUCKET, - Key=self.get_thumbnail_key(video_id, '00001', 'png'), + Key=self.get_thumbnail_key(video_id, "00001", "png"), ) - initial_thumbnail_file = response['Body'] - initial_thumbnail_file.name = '00001.png' + initial_thumbnail_file = response["Body"] + initial_thumbnail_file.name = "00001.png" # Convert it to jpg - thumbnail_file = NamedTemporaryFile(mode='rb', suffix=".jpg") + thumbnail_file = NamedTemporaryFile(mode="rb", suffix=".jpg") pipeline.utils.make_thumbnail(initial_thumbnail_file, thumbnail_file.name) # Upload jpg thumbnail @@ -234,18 +237,27 @@ def delete_thumbnail(self, video_id, thumb_id): pass def video_url(self, public_video_id, format_name): - return self._get_download_base_url() + '/' + self.VIDEO_KEY_PATTERN.format( - video_id=public_video_id, - resolution=format_name, + return ( + self._get_download_base_url() + + "/" + + self.VIDEO_KEY_PATTERN.format( + video_id=public_video_id, resolution=format_name + ) ) def subtitle_url(self, video_id, subtitle_id, language): - return self._get_download_base_url() + '/' + self.SUBTITLE_KEY_PATTERN.format( - video_id=video_id, - subtitle_id=subtitle_id, - language=language, + return ( + self._get_download_base_url() + + "/" + + self.SUBTITLE_KEY_PATTERN.format( + video_id=video_id, subtitle_id=subtitle_id, language=language + ) ) def thumbnail_url(self, video_id, thumb_id): # Use the first generated thumbnail as the video thumbnail - return self._get_download_base_url() + '/' + self.get_thumbnail_key(video_id, thumb_id) + return ( + self._get_download_base_url() + + "/" + + self.get_thumbnail_key(video_id, thumb_id) + ) diff --git a/contrib/plugins/aws/management/commands/bootstrap-s3.py b/contrib/plugins/aws/management/commands/bootstrap-s3.py index 3f0ec57..9c8a33e 100644 --- a/contrib/plugins/aws/management/commands/bootstrap-s3.py +++ b/contrib/plugins/aws/management/commands/bootstrap-s3.py @@ -1,4 +1,5 @@ from botocore.exceptions import ClientError + from django.conf import settings from django.core.management.base import BaseCommand @@ -6,13 +7,15 @@ class Command(BaseCommand): - help = 'Bootstrap S3 for video file storage' + help = "Bootstrap S3 for video file storage" def handle(self, *args, **options): - acl = 'private' if hasattr(settings, 'CLOUDFRONT_DOMAIN_NAME') else 'public-read' + acl = ( + "private" if hasattr(settings, "CLOUDFRONT_DOMAIN_NAME") else "public-read" + ) self.create_bucket(settings.S3_BUCKET, acl) - self.create_bucket(settings.S3_PRIVATE_BUCKET, 'private') + self.create_bucket(settings.S3_PRIVATE_BUCKET, "private") def create_bucket(self, bucket_name, acl): backend = Backend() @@ -24,27 +27,19 @@ def create_bucket(self, bucket_name, acl): backend.s3_client.create_bucket( ACL=acl, Bucket=bucket_name, - CreateBucketConfiguration={ - 'LocationConstraint': settings.AWS_REGION - } + CreateBucketConfiguration={"LocationConstraint": settings.AWS_REGION}, ) self.stdout.write("Updating CORS configuration...") backend.s3_client.put_bucket_cors( Bucket=bucket_name, CORSConfiguration={ - 'CORSRules': [ + "CORSRules": [ { - 'AllowedHeaders': [ - '*', - ], - 'AllowedMethods': [ - 'GET', 'PUT', - ], - 'AllowedOrigins': [ - '*', - ], - 'MaxAgeSeconds': 3000 - }, + "AllowedHeaders": ["*"], + "AllowedMethods": ["GET", "PUT"], + "AllowedOrigins": ["*"], + "MaxAgeSeconds": 3000, + } ] - } + }, ) diff --git a/contrib/plugins/aws/management/commands/delete-s3-folders.py b/contrib/plugins/aws/management/commands/delete-s3-folders.py index bd759c7..3ad2d8b 100644 --- a/contrib/plugins/aws/management/commands/delete-s3-folders.py +++ b/contrib/plugins/aws/management/commands/delete-s3-folders.py @@ -4,12 +4,12 @@ class Command(BaseCommand): - help = 'Delete one or more folders on S3' + help = "Delete one or more folders on S3" def add_arguments(self, parser): - parser.add_argument('folders', nargs='+', help='Folder names') + parser.add_argument("folders", nargs="+", help="Folder names") def handle(self, *args, **options): backend = Backend() - for folder in options['folders']: + for folder in options["folders"]: backend.delete_objects(folder) diff --git a/contrib/plugins/aws/tests/test_backend.py b/contrib/plugins/aws/tests/test_backend.py index de5c5d0..0c1df8a 100644 --- a/contrib/plugins/aws/tests/test_backend.py +++ b/contrib/plugins/aws/tests/test_backend.py @@ -3,263 +3,287 @@ from unittest.mock import Mock, patch from botocore.exceptions import ClientError + from django.test import TestCase from django.test.utils import override_settings +from contrib.plugins.aws import backend as aws_backend import pipeline.backend import pipeline.exceptions import pipeline.tasks from pipeline.tests.factories import VideoFactory -from contrib.plugins.aws import backend as aws_backend + from . import utils @utils.override_s3_settings class VideoUploadUrlTests(TestCase): - def test_upload_video(self): backend = aws_backend.Backend() backend._s3_client = Mock(put_object=Mock()) file_object = Mock() file_object.name = "somevideo.mp4" - backend.upload_video('videoid', file_object) + backend.upload_video("videoid", file_object) backend.s3_client.put_object.assert_called_once() - self.assertEqual("private", backend.s3_client.put_object.call_args[1]['ACL']) - self.assertEqual("videos/videoid/src/somevideo.mp4", backend.s3_client.put_object.call_args[1]['Key']) - self.assertEqual("privates3bucket", backend.s3_client.put_object.call_args[1]['Bucket']) + self.assertEqual("private", backend.s3_client.put_object.call_args[1]["ACL"]) + self.assertEqual( + "videos/videoid/src/somevideo.mp4", + backend.s3_client.put_object.call_args[1]["Key"], + ) + self.assertEqual( + "privates3bucket", backend.s3_client.put_object.call_args[1]["Bucket"] + ) def test_delete_video_no_content(self): backend = aws_backend.Backend() backend._s3_client = Mock(list_objects=Mock(return_value={})) - backend.delete_video('videoid') + backend.delete_video("videoid") backend.s3_client.list_objects.assert_any_call( - Bucket='privates3bucket', Prefix='videos/videoid/' + Bucket="privates3bucket", Prefix="videos/videoid/" ) backend.s3_client.list_objects.assert_any_call( - Bucket='publics3bucket', Prefix='videos/videoid/' + Bucket="publics3bucket", Prefix="videos/videoid/" ) def test_delete_subtitle(self): backend = aws_backend.Backend() backend._s3_client = Mock(list_objects=Mock(return_value={})) - backend.delete_subtitle('videoid', 'subid') + backend.delete_subtitle("videoid", "subid") backend.s3_client.list_objects.assert_any_call( - Bucket='publics3bucket', Prefix='videos/videoid/subs/subid.' + Bucket="publics3bucket", Prefix="videos/videoid/subs/subid." ) - @override_settings(PLUGIN_BACKEND='contrib.plugins.aws.backend.Backend') + @override_settings(PLUGIN_BACKEND="contrib.plugins.aws.backend.Backend") def test_video_url(self): backend = pipeline.backend.get() - url = backend.video_url('videoid', 'SD') + url = backend.video_url("videoid", "SD") self.assertIsNotNone(url) self.assertTrue(url.startswith("https://s3-dummyawsregion")) @override_settings( - PLUGIN_BACKEND='contrib.plugins.aws.backend.Backend', - CLOUDFRONT_DOMAIN_NAME='cloudfrontid.cloudfront.net' + PLUGIN_BACKEND="contrib.plugins.aws.backend.Backend", + CLOUDFRONT_DOMAIN_NAME="cloudfrontid.cloudfront.net", ) def test_video_url_with_cloudfront(self): backend = pipeline.backend.get() - url = backend.video_url('videoid', 'SD') - self.assertEqual("https://cloudfrontid.cloudfront.net/videos/videoid/SD.mp4", url) + url = backend.video_url("videoid", "SD") + self.assertEqual( + "https://cloudfrontid.cloudfront.net/videos/videoid/SD.mp4", url + ) @utils.override_s3_settings class TranscodeTests(TestCase): - @override_settings( - ELASTIC_TRANSCODER_PIPELINE_ID='pipelineid', - ELASTIC_TRANSCODER_PRESETS=[('SD', 'presetid', 128)], - ELASTIC_TRANSCODER_THUMBNAILS_PRESET='thumbspresetid' + ELASTIC_TRANSCODER_PIPELINE_ID="pipelineid", + ELASTIC_TRANSCODER_PRESETS=[("SD", "presetid", 128)], + ELASTIC_TRANSCODER_THUMBNAILS_PRESET="thumbspresetid", ) def test_start_transcoding(self): - create_job_fixture = utils.load_json_fixture('elastictranscoder_create_job.json') + create_job_fixture = utils.load_json_fixture( + "elastictranscoder_create_job.json" + ) backend = aws_backend.Backend() - backend.get_src_file_key = Mock(return_value='videos/videoid/src/Some video file.mpg') + backend.get_src_file_key = Mock( + return_value="videos/videoid/src/Some video file.mpg" + ) backend._elastictranscoder_client = Mock( - create_job=Mock(return_value=create_job_fixture), + create_job=Mock(return_value=create_job_fixture) ) - jobs = backend.start_transcoding('videoid') + jobs = backend.start_transcoding("videoid") self.assertEqual(1, len(jobs)) backend.elastictranscoder_client.create_job.assert_called_once_with( - PipelineId='pipelineid', - Input={'Key': 'videos/videoid/src/Some video file.mpg'}, - Output={'PresetId': 'presetid', 'Key': 'videos/videoid/SD.mp4'} + PipelineId="pipelineid", + Input={"Key": "videos/videoid/src/Some video file.mpg"}, + Output={"PresetId": "presetid", "Key": "videos/videoid/SD.mp4"}, ) - backend.get_src_file_key.assert_called_once_with('videoid') + backend.get_src_file_key.assert_called_once_with("videoid") @override_settings( - ELASTIC_TRANSCODER_PIPELINE_ID='pipelineid', - ELASTIC_TRANSCODER_PRESETS=[('SD', 'sdpresetid', 128), ('HD', 'hdpresetid', 256)], - ELASTIC_TRANSCODER_THUMBNAILS_PRESET='hdpresetid' + ELASTIC_TRANSCODER_PIPELINE_ID="pipelineid", + ELASTIC_TRANSCODER_PRESETS=[ + ("SD", "sdpresetid", 128), + ("HD", "hdpresetid", 256), + ], + ELASTIC_TRANSCODER_THUMBNAILS_PRESET="hdpresetid", ) def test_start_transcoding_with_thumbnails(self): - create_job_fixture = utils.load_json_fixture('elastictranscoder_create_job.json') + create_job_fixture = utils.load_json_fixture( + "elastictranscoder_create_job.json" + ) backend = aws_backend.Backend() - backend.get_src_file_key = Mock(return_value='videos/videoid/src/Some video file.mpg') + backend.get_src_file_key = Mock( + return_value="videos/videoid/src/Some video file.mpg" + ) backend._elastictranscoder_client = Mock( - create_job=Mock(return_value=create_job_fixture), + create_job=Mock(return_value=create_job_fixture) ) - backend.start_transcoding('videoid') + backend.start_transcoding("videoid") # SD + Thumbnails backend.elastictranscoder_client.create_job.assert_any_call( - PipelineId='pipelineid', - Input={'Key': 'videos/videoid/src/Some video file.mpg'}, - Output={ - 'PresetId': 'sdpresetid', - 'Key': 'videos/videoid/SD.mp4' - }, + PipelineId="pipelineid", + Input={"Key": "videos/videoid/src/Some video file.mpg"}, + Output={"PresetId": "sdpresetid", "Key": "videos/videoid/SD.mp4"}, ) # HD + Thumbnails backend.elastictranscoder_client.create_job.assert_any_call( - PipelineId='pipelineid', - Input={'Key': 'videos/videoid/src/Some video file.mpg'}, + PipelineId="pipelineid", + Input={"Key": "videos/videoid/src/Some video file.mpg"}, Output={ - 'PresetId': 'hdpresetid', - 'Key': 'videos/videoid/HD.mp4', - 'ThumbnailPattern': 'videos/videoid/thumbs/{count}' + "PresetId": "hdpresetid", + "Key": "videos/videoid/HD.mp4", + "ThumbnailPattern": "videos/videoid/thumbs/{count}", }, ) def test_check_progress(self): - job = utils.load_json_fixture('elastictranscoder_create_job.json') - read_job_fixture = utils.load_json_fixture('elastictranscoder_read_job_complete.json') + job = utils.load_json_fixture("elastictranscoder_create_job.json") + read_job_fixture = utils.load_json_fixture( + "elastictranscoder_read_job_complete.json" + ) backend = aws_backend.Backend() backend._elastictranscoder_client = Mock( - read_job=Mock(return_value=read_job_fixture), + read_job=Mock(return_value=read_job_fixture) ) - progress, finished = backend.check_progress(job['Job']) + progress, finished = backend.check_progress(job["Job"]) self.assertEqual(100, progress) self.assertTrue(finished) backend.elastictranscoder_client.read_job.assert_called_once_with( - Id='jobid' # job id in test fixture + Id="jobid" # job id in test fixture ) @override_settings( - ELASTIC_TRANSCODER_PIPELINE_ID='pipelineid', - ELASTIC_TRANSCODER_PRESETS=[('SD', 'presetid', 128)], - ELASTIC_TRANSCODER_THUMBNAILS_PRESET='thumbspresetid' + ELASTIC_TRANSCODER_PIPELINE_ID="pipelineid", + ELASTIC_TRANSCODER_PRESETS=[("SD", "presetid", 128)], + ELASTIC_TRANSCODER_THUMBNAILS_PRESET="thumbspresetid", ) def test_transcoding_pipeline_compatibility(self): - create_job_fixture = utils.load_json_fixture('elastictranscoder_create_job.json') - read_job_fixture = utils.load_json_fixture('elastictranscoder_read_job_complete.json') + create_job_fixture = utils.load_json_fixture( + "elastictranscoder_create_job.json" + ) + read_job_fixture = utils.load_json_fixture( + "elastictranscoder_read_job_complete.json" + ) backend = aws_backend.Backend() - backend.get_src_file_key = Mock(return_value='videos/videoid/src/Some video file.mpg') + backend.get_src_file_key = Mock( + return_value="videos/videoid/src/Some video file.mpg" + ) backend._elastictranscoder_client = Mock( create_job=Mock(return_value=create_job_fixture), - read_job=Mock(return_value=read_job_fixture) + read_job=Mock(return_value=read_job_fixture), ) - jobs = backend.start_transcoding('videoid') + jobs = backend.start_transcoding("videoid") backend.check_progress(jobs[0]) - @override_settings(ELASTIC_TRANSCODER_PRESETS=[('SD', 'presetid1', 128), ('HD', 'presetid2', 256)]) + @override_settings( + ELASTIC_TRANSCODER_PRESETS=[("SD", "presetid1", 128), ("HD", "presetid2", 256)] + ) def test_iter_formats(self): backend = aws_backend.Backend() def head_object(Bucket=None, Key=None): - if Key != 'videos/videoid/HD.mp4': - raise ClientError({'Error': {}}, 'head_object') + if Key != "videos/videoid/HD.mp4": + raise ClientError({"Error": {}}, "head_object") backend.s3_client.head_object = head_object - formats = list(backend.iter_formats('videoid')) + formats = list(backend.iter_formats("videoid")) - self.assertEqual([('HD', 256)], formats) + self.assertEqual([("HD", 256)], formats) @utils.override_s3_settings class ThumbnailsTests(TestCase): - def setUp(self): VideoFactory(public_id="videoid") - @patch('pipeline.utils.resize_image') - @patch('contrib.plugins.aws.backend.Backend.s3_client') + @patch("pipeline.utils.resize_image") + @patch("contrib.plugins.aws.backend.Backend.s3_client") def test_create_thumbnail(self, mock_s3_client, mock_resize_image): thumbnail_file = BytesIO(b"") - mock_s3_client.get_object = Mock(return_value={ - 'Body': thumbnail_file - }) + mock_s3_client.get_object = Mock(return_value={"Body": thumbnail_file}) backend = aws_backend.Backend() - backend.create_thumbnail('videoid', 'thumbid') + backend.create_thumbnail("videoid", "thumbid") mock_s3_client.get_object.assert_called_once() - self.assertEqual('videos/videoid/thumbs/00001.png', mock_s3_client.get_object.call_args[1]['Key']) + self.assertEqual( + "videos/videoid/thumbs/00001.png", + mock_s3_client.get_object.call_args[1]["Key"], + ) mock_resize_image.assert_called_once() def test_thumbnail_url(self): backend = aws_backend.Backend() - thumbnail_url = backend.thumbnail_url('videoid', 'thumbid') + thumbnail_url = backend.thumbnail_url("videoid", "thumbid") self.assertIsNotNone(thumbnail_url) - self.assertIn('videoid', thumbnail_url) - self.assertIn('thumbid.jpg', thumbnail_url) + self.assertIn("videoid", thumbnail_url) + self.assertIn("thumbid.jpg", thumbnail_url) - @override_settings(PLUGIN_BACKEND='contrib.plugins.aws.backend.Backend') - @patch('contrib.plugins.aws.backend.Backend.s3_client') + @override_settings(PLUGIN_BACKEND="contrib.plugins.aws.backend.Backend") + @patch("contrib.plugins.aws.backend.Backend.s3_client") def test_thumbnail_compatibility(self, mock_s3_client): - def mock_resize_image(in_path, out_path, max_size): # Mock resize just copies the content from the source path to the # destination path self.assertIn(".jpg", out_path) shutil.copy(in_path, out_path) - thumb_file = BytesIO(b'\x89PNG\r\n\x1a\n') + thumb_file = BytesIO(b"\x89PNG\r\n\x1a\n") thumb_file.name = "thumb.png" - with patch('pipeline.utils.resize_image', mock_resize_image): - pipeline.tasks.upload_thumbnail('videoid', thumb_file) + with patch("pipeline.utils.resize_image", mock_resize_image): + pipeline.tasks.upload_thumbnail("videoid", thumb_file) mock_s3_client.put_object.assert_called_once() - @override_settings(PLUGIN_BACKEND='contrib.plugins.aws.backend.Backend') - @patch('contrib.plugins.aws.backend.Backend.s3_client') + @override_settings(PLUGIN_BACKEND="contrib.plugins.aws.backend.Backend") + @patch("contrib.plugins.aws.backend.Backend.s3_client") def test_delete_thumbnail(self, mock_s3_client): backend = aws_backend.Backend() - backend.delete_thumbnail('videoid', 'thumbid') + backend.delete_thumbnail("videoid", "thumbid") mock_s3_client.delete_object.assert_called_once_with( Bucket="publics3bucket", Key="videos/videoid/thumbs/thumbid.jpg" ) + @utils.override_s3_settings class SubtitleTest(TestCase): - - @override_settings(PLUGIN_BACKEND='contrib.plugins.aws.backend.Backend') - @patch('contrib.plugins.aws.backend.Backend.s3_client') + @override_settings(PLUGIN_BACKEND="contrib.plugins.aws.backend.Backend") + @patch("contrib.plugins.aws.backend.Backend.s3_client") def test_upload_subtitle_compatibility(self, mock_s3_client): - pipeline.tasks.upload_subtitle('videoid', 'subid', 'fr', b"WEBVTT") + pipeline.tasks.upload_subtitle("videoid", "subid", "fr", b"WEBVTT") mock_s3_client.put_object.assert_called_once() - @override_settings(PLUGIN_BACKEND='contrib.plugins.aws.backend.Backend') + @override_settings(PLUGIN_BACKEND="contrib.plugins.aws.backend.Backend") def test_subtitle_url_compatibility(self): - video = VideoFactory(public_id='videoid') - subtitle = video.subtitles.create(language='fr') + video = VideoFactory(public_id="videoid") + subtitle = video.subtitles.create(language="fr") self.assertIsNotNone(subtitle.url) def subtitle_url(self): backend = aws_backend.Backend() - url = backend.subtitle_url('videoid', 'subid', 'uk') + url = backend.subtitle_url("videoid", "subid", "uk") self.assertIsNotNone(url) - self.assertIn('videoid', url) - self.assertIn('subid', url) - self.assertIn('uk', url) + self.assertIn("videoid", url) + self.assertIn("subid", url) + self.assertIn("uk", url) - @override_settings(CLOUDFRONT_DOMAIN_NAME='cloudfrontid.cloudfront.net') + @override_settings(CLOUDFRONT_DOMAIN_NAME="cloudfrontid.cloudfront.net") def test_subtitle_url(self): backend = aws_backend.Backend() - url = backend.subtitle_url('videoid', 'subid', 'uk') - self.assertTrue(url.startswith('https://cloudfrontid.cloudfront.net')) + url = backend.subtitle_url("videoid", "subid", "uk") + self.assertTrue(url.startswith("https://cloudfrontid.cloudfront.net")) diff --git a/contrib/plugins/aws/tests/utils.py b/contrib/plugins/aws/tests/utils.py index 2104511..ffead0f 100644 --- a/contrib/plugins/aws/tests/utils.py +++ b/contrib/plugins/aws/tests/utils.py @@ -3,18 +3,20 @@ from django.test.utils import override_settings + override_s3_settings = override_settings( - AWS_ACCESS_KEY_ID='dummyawsaccesskey', - AWS_SECRET_ACCESS_KEY='dummyawssecretkey', - AWS_REGION='dummyawsregion', - S3_PRIVATE_BUCKET='privates3bucket', - S3_BUCKET='publics3bucket', + AWS_ACCESS_KEY_ID="dummyawsaccesskey", + AWS_SECRET_ACCESS_KEY="dummyawssecretkey", + AWS_REGION="dummyawsregion", + S3_PRIVATE_BUCKET="privates3bucket", + S3_BUCKET="publics3bucket", ) + def load_json_fixture(name): """ Load a json fixtures file from the 'fixtures/' directory. """ - directory = os.path.join(os.path.dirname(__file__), 'fixtures') + directory = os.path.join(os.path.dirname(__file__), "fixtures") fixture_path = os.path.join(directory, name) return json.load(open(fixture_path)) diff --git a/manage.py b/manage.py index 44cc4b6..3ef041c 100755 --- a/manage.py +++ b/manage.py @@ -2,6 +2,7 @@ import os import sys + if __name__ == "__main__": os.environ.setdefault("DJANGO_SETTINGS_MODULE", "videofront.settings") diff --git a/pipeline/admin.py b/pipeline/admin.py index 6ffcc0d..52f72e3 100644 --- a/pipeline/admin.py +++ b/pipeline/admin.py @@ -9,52 +9,53 @@ class ProcessingStateInlineAdmin(admin.TabularInline): class VideoAdmin(admin.ModelAdmin): list_display = ( - 'public_id', 'title', 'owner', - 'processing_progress', 'processing_status', 'processing_started_at', + "public_id", + "title", + "owner", + "processing_progress", + "processing_status", + "processing_started_at", ) - search_fields = ('title', 'public_id',) - list_filter = ('owner',) - raw_id_fields = ('owner',) + search_fields = ("title", "public_id") + list_filter = ("owner",) + raw_id_fields = ("owner",) inlines = [ProcessingStateInlineAdmin] def get_queryset(self, request): qs = super(VideoAdmin, self).get_queryset(request) - return qs.select_related('processing_state') + return qs.select_related("processing_state") class VideoUploadUrlAdmin(admin.ModelAdmin): model = models.VideoUploadUrl - list_display = ('public_video_id', 'owner', 'expires_at', 'was_used',) - list_filter = ('owner',) - raw_id_fields = ('owner',) - search_fields = ('public_video_id',) + list_display = ("public_video_id", "owner", "expires_at", "was_used") + list_filter = ("owner",) + raw_id_fields = ("owner",) + search_fields = ("public_video_id",) class PlaylistAdmin(admin.ModelAdmin): model = models.Playlist - list_display = ('public_id', 'name', 'owner',) - list_filter = ('owner',) - raw_id_fields = ('owner',) - search_fields = ( - 'public_id', 'name', 'videos__public_id', - 'videos__title' - ) - filter_horizontal = ('videos',) + list_display = ("public_id", "name", "owner") + list_filter = ("owner",) + raw_id_fields = ("owner",) + search_fields = ("public_id", "name", "videos__public_id", "videos__title") + filter_horizontal = ("videos",) class SubtitleAdmin(admin.ModelAdmin): model = models.Subtitle - list_display = ('public_id', 'video', 'language') - list_filter = ('language',) - raw_id_fields = ('video',) - search_fields = ('public_id', 'video__public_id', 'video__title') + list_display = ("public_id", "video", "language") + list_filter = ("language",) + raw_id_fields = ("video",) + search_fields = ("public_id", "video__public_id", "video__title") class VideoFormatAdmin(admin.ModelAdmin): model = models.VideoFormat - list_display = ('__str__', 'name', 'video', 'bitrate') - raw_id_fields = ('video',) - search_fields = ('name', 'bitrate', 'video__public_id', 'video__title') + list_display = ("__str__", "name", "video", "bitrate") + raw_id_fields = ("video",) + search_fields = ("name", "bitrate", "video__public_id", "video__title") admin.site.register(models.Video, VideoAdmin) diff --git a/pipeline/apps.py b/pipeline/apps.py index 68b4c21..d938611 100644 --- a/pipeline/apps.py +++ b/pipeline/apps.py @@ -2,4 +2,4 @@ class PipelineConfig(AppConfig): - name = 'pipeline' + name = "pipeline" diff --git a/pipeline/backend.py b/pipeline/backend.py index f16ace2..2bba1f1 100644 --- a/pipeline/backend.py +++ b/pipeline/backend.py @@ -1,9 +1,9 @@ import importlib + from django.conf import settings class BaseBackend(object): - def upload_video(self, video_id, file_object): """ Store a video file for transcoding. @@ -138,7 +138,7 @@ def thumbnail_url(self, video_id, thumb_id): This feature is optional. If undefined, the thumbnail url will be an empty string. """ - return '' + return "" class UndefinedPluginBackend(Exception): @@ -159,11 +159,11 @@ def get(): MissingPluginBackend in case of a missing plugin class definition """ - setting = getattr(settings, 'PLUGIN_BACKEND') + setting = getattr(settings, "PLUGIN_BACKEND") if setting is None: raise UndefinedPluginBackend() - if hasattr(setting, '__call__'): + if hasattr(setting, "__call__"): backend_object = setting() else: module_name, object_name = setting.rsplit(".", 1) diff --git a/pipeline/cache.py b/pipeline/cache.py index 259c730..8324cb0 100644 --- a/pipeline/cache.py +++ b/pipeline/cache.py @@ -5,6 +5,7 @@ VIDEO_CACHE_TIMEOUT = 3600 + def _cache_key(public_id): """ Key which stores the video content in the cache. Of course, the cache @@ -16,11 +17,13 @@ def _cache_key(public_id): def invalidate(public_video_id): cache.delete(_cache_key(public_video_id)) + def get(public_video_id): content = cache.get(_cache_key(public_video_id)) if content is not None: return json.loads(content) return None + def set(public_video_id, data): return cache.set(_cache_key(public_video_id), json.dumps(data), VIDEO_CACHE_TIMEOUT) diff --git a/pipeline/exceptions.py b/pipeline/exceptions.py index 19b4d85..6afc75f 100644 --- a/pipeline/exceptions.py +++ b/pipeline/exceptions.py @@ -2,6 +2,7 @@ class LockUnavailable(Exception): """ Raised whenever we try to acquire a lock that was already acquired. """ + pass @@ -9,6 +10,7 @@ class TranscodingFailed(Exception): """ Raised whenever a transcoding task failed. """ + pass @@ -16,6 +18,7 @@ class SubtitleInvalid(Exception): """ Raised whenever subtitle cannot be converted to utf8 or to VTT format. """ + pass @@ -23,4 +26,5 @@ class ThumbnailInvalid(Exception): """ Raised whenever the thumbnail file is invalid """ + pass diff --git a/pipeline/management/commands/createuser.py b/pipeline/management/commands/createuser.py index 8055bc6..b2f5f7b 100644 --- a/pipeline/management/commands/createuser.py +++ b/pipeline/management/commands/createuser.py @@ -1,28 +1,28 @@ -from django.core.management.base import BaseCommand from django.contrib.auth.models import User +from django.core.management.base import BaseCommand class Command(BaseCommand): - help = 'Create a user with the given password. If the user already exists, it is updated.' + help = "Create a user with the given password. If the user already exists, it is updated." def add_arguments(self, parser): - parser.add_argument('--admin', action='store_true', help='Make the user admin') - parser.add_argument('username', help='Authentication username') - parser.add_argument('password', help='Authentication password') + parser.add_argument("--admin", action="store_true", help="Make the user admin") + parser.add_argument("username", help="Authentication username") + parser.add_argument("password", help="Authentication password") def handle(self, *args, **options): - username = options['username'] - password = options['password'] + username = options["username"] + password = options["password"] user, created = User.objects.get_or_create(username=username) user.set_password(password) - if options.get('admin'): + if options.get("admin"): user.is_superuser = True user.is_staff = True user.save() - self.stdout.write("{} user '{}' with token: {}".format( - "Created" if created else "Updated", - username, - user.auth_token.key - )) + self.stdout.write( + "{} user '{}' with token: {}".format( + "Created" if created else "Updated", username, user.auth_token.key + ) + ) diff --git a/pipeline/management/commands/transcode-video.py b/pipeline/management/commands/transcode-video.py index 4dd2aef..9ac5504 100644 --- a/pipeline/management/commands/transcode-video.py +++ b/pipeline/management/commands/transcode-video.py @@ -4,12 +4,12 @@ class Command(BaseCommand): - help = 'Force a transcoding task to start.' + help = "Force a transcoding task to start." def add_arguments(self, parser): - parser.add_argument('video_id', help='Public video ID') + parser.add_argument("video_id", help="Public video ID") def handle(self, *args, **options): - public_video_id = options['video_id'] - send_task('transcode_video', args=(public_video_id,)) + public_video_id = options["video_id"] + send_task("transcode_video", args=(public_video_id,)) self.stdout.write("Done.") diff --git a/pipeline/managers.py b/pipeline/managers.py index e08d446..98fb353 100644 --- a/pipeline/managers.py +++ b/pipeline/managers.py @@ -19,4 +19,6 @@ def obsolete(self): """ Unused upload urls that have expired. """ - return self.filter(expires_at__lt=time() - 2*self.EXPIRE_DELAY, was_used=False) + return self.filter( + expires_at__lt=time() - 2 * self.EXPIRE_DELAY, was_used=False + ) diff --git a/pipeline/migrations/0001_initial.py b/pipeline/migrations/0001_initial.py index dde73e0..3d91e77 100644 --- a/pipeline/migrations/0001_initial.py +++ b/pipeline/migrations/0001_initial.py @@ -5,6 +5,7 @@ import django.core.validators from django.db import migrations, models import django.db.models.deletion + import pipeline.utils @@ -12,74 +13,342 @@ class Migration(migrations.Migration): initial = True - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] + dependencies = [migrations.swappable_dependency(settings.AUTH_USER_MODEL)] operations = [ migrations.CreateModel( - name='Playlist', + name="Playlist", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(db_index=True, max_length=128)), - ('public_id', models.CharField(default=pipeline.utils.generate_random_id, max_length=20, null=True, unique=True, validators=[django.core.validators.MinLengthValidator(1)])), - ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(db_index=True, max_length=128)), + ( + "public_id", + models.CharField( + default=pipeline.utils.generate_random_id, + max_length=20, + null=True, + unique=True, + validators=[django.core.validators.MinLengthValidator(1)], + ), + ), + ( + "owner", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), ], ), migrations.CreateModel( - name='Video', + name="Video", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('title', models.CharField(max_length=100)), - ('public_id', models.CharField(default=pipeline.utils.generate_random_id, max_length=20, null=True, unique=True, validators=[django.core.validators.MinLengthValidator(1)])), - ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("title", models.CharField(max_length=100)), + ( + "public_id", + models.CharField( + default=pipeline.utils.generate_random_id, + max_length=20, + null=True, + unique=True, + validators=[django.core.validators.MinLengthValidator(1)], + ), + ), + ( + "owner", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), ], ), migrations.CreateModel( - name='VideoFormat', + name="VideoFormat", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=128)), - ('bitrate', models.FloatField(validators=[django.core.validators.MinValueValidator(0)])), - ('video', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='formats', to='pipeline.Video')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=128)), + ( + "bitrate", + models.FloatField( + validators=[django.core.validators.MinValueValidator(0)] + ), + ), + ( + "video", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="formats", + to="pipeline.Video", + ), + ), ], ), migrations.CreateModel( - name='VideoSubtitles', + name="VideoSubtitles", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('public_id', models.CharField(default=pipeline.utils.generate_random_id, max_length=20, null=True, unique=True, validators=[django.core.validators.MinLengthValidator(1)])), - ('language', models.CharField(choices=[('af', 'Afrikaans'), ('ar', 'Arabic'), ('az', 'Azerbaijani'), ('bg', 'Bulgarian'), ('be', 'Belarusian'), ('bn', 'Bengali'), ('br', 'Breton'), ('bs', 'Bosnian'), ('ca', 'Catalan'), ('cs', 'Czech'), ('cy', 'Welsh'), ('da', 'Danish'), ('de', 'German'), ('el', 'Greek'), ('en', 'English'), ('eo', 'Esperanto'), ('es', 'Spanish'), ('et', 'Estonian'), ('eu', 'Basque'), ('fa', 'Persian'), ('fi', 'Finnish'), ('fr', 'French'), ('fy', 'Frisian'), ('ga', 'Irish'), ('gd', 'Scottish Gaelic'), ('gl', 'Galician'), ('he', 'Hebrew'), ('hi', 'Hindi'), ('hr', 'Croatian'), ('hu', 'Hungarian'), ('ia', 'Interlingua'), ('id', 'Indonesian'), ('io', 'Ido'), ('is', 'Icelandic'), ('it', 'Italian'), ('ja', 'Japanese'), ('ka', 'Georgian'), ('kk', 'Kazakh'), ('km', 'Khmer'), ('kn', 'Kannada'), ('ko', 'Korean'), ('lb', 'Luxembourgish'), ('lt', 'Lithuanian'), ('lv', 'Latvian'), ('mk', 'Macedonian'), ('ml', 'Malayalam'), ('mn', 'Mongolian'), ('mr', 'Marathi'), ('my', 'Burmese'), ('nb', 'Norwegian Bokmal'), ('ne', 'Nepali'), ('nl', 'Dutch'), ('nn', 'Norwegian Nynorsk'), ('os', 'Ossetic'), ('pa', 'Punjabi'), ('pl', 'Polish'), ('pt', 'Portuguese'), ('ro', 'Romanian'), ('ru', 'Russian'), ('sk', 'Slovak'), ('sl', 'Slovenian'), ('sq', 'Albanian'), ('sr', 'Serbian'), ('sv', 'Swedish'), ('sw', 'Swahili'), ('ta', 'Tamil'), ('te', 'Telugu'), ('th', 'Thai'), ('tr', 'Turkish'), ('tt', 'Tatar'), ('uk', 'Ukrainian'), ('ur', 'Urdu'), ('vi', 'Vietnamese')], max_length=2, null=True, validators=[django.core.validators.MinLengthValidator(2)])), - ('video', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='subtitles', to='pipeline.Video')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "public_id", + models.CharField( + default=pipeline.utils.generate_random_id, + max_length=20, + null=True, + unique=True, + validators=[django.core.validators.MinLengthValidator(1)], + ), + ), + ( + "language", + models.CharField( + choices=[ + ("af", "Afrikaans"), + ("ar", "Arabic"), + ("az", "Azerbaijani"), + ("bg", "Bulgarian"), + ("be", "Belarusian"), + ("bn", "Bengali"), + ("br", "Breton"), + ("bs", "Bosnian"), + ("ca", "Catalan"), + ("cs", "Czech"), + ("cy", "Welsh"), + ("da", "Danish"), + ("de", "German"), + ("el", "Greek"), + ("en", "English"), + ("eo", "Esperanto"), + ("es", "Spanish"), + ("et", "Estonian"), + ("eu", "Basque"), + ("fa", "Persian"), + ("fi", "Finnish"), + ("fr", "French"), + ("fy", "Frisian"), + ("ga", "Irish"), + ("gd", "Scottish Gaelic"), + ("gl", "Galician"), + ("he", "Hebrew"), + ("hi", "Hindi"), + ("hr", "Croatian"), + ("hu", "Hungarian"), + ("ia", "Interlingua"), + ("id", "Indonesian"), + ("io", "Ido"), + ("is", "Icelandic"), + ("it", "Italian"), + ("ja", "Japanese"), + ("ka", "Georgian"), + ("kk", "Kazakh"), + ("km", "Khmer"), + ("kn", "Kannada"), + ("ko", "Korean"), + ("lb", "Luxembourgish"), + ("lt", "Lithuanian"), + ("lv", "Latvian"), + ("mk", "Macedonian"), + ("ml", "Malayalam"), + ("mn", "Mongolian"), + ("mr", "Marathi"), + ("my", "Burmese"), + ("nb", "Norwegian Bokmal"), + ("ne", "Nepali"), + ("nl", "Dutch"), + ("nn", "Norwegian Nynorsk"), + ("os", "Ossetic"), + ("pa", "Punjabi"), + ("pl", "Polish"), + ("pt", "Portuguese"), + ("ro", "Romanian"), + ("ru", "Russian"), + ("sk", "Slovak"), + ("sl", "Slovenian"), + ("sq", "Albanian"), + ("sr", "Serbian"), + ("sv", "Swedish"), + ("sw", "Swahili"), + ("ta", "Tamil"), + ("te", "Telugu"), + ("th", "Thai"), + ("tr", "Turkish"), + ("tt", "Tatar"), + ("uk", "Ukrainian"), + ("ur", "Urdu"), + ("vi", "Vietnamese"), + ], + max_length=2, + null=True, + validators=[django.core.validators.MinLengthValidator(2)], + ), + ), + ( + "video", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="subtitles", + to="pipeline.Video", + ), + ), ], ), migrations.CreateModel( - name='VideoTranscoding', + name="VideoTranscoding", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('started_at', models.DateTimeField(auto_now=True, verbose_name='Time of transcoding job start')), - ('progress', models.FloatField(default=0, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(100)], verbose_name='Progress percentage')), - ('status', models.CharField(choices=[('pending', 'Pending'), ('processing', 'Processing'), ('failed', 'Failed'), ('success', 'Success')], max_length=32, verbose_name='Status')), - ('message', models.CharField(blank=True, max_length=1024)), - ('video', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='transcoding', to='pipeline.Video')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "started_at", + models.DateTimeField( + auto_now=True, verbose_name="Time of transcoding job start" + ), + ), + ( + "progress", + models.FloatField( + default=0, + validators=[ + django.core.validators.MinValueValidator(0), + django.core.validators.MaxValueValidator(100), + ], + verbose_name="Progress percentage", + ), + ), + ( + "status", + models.CharField( + choices=[ + ("pending", "Pending"), + ("processing", "Processing"), + ("failed", "Failed"), + ("success", "Success"), + ], + max_length=32, + verbose_name="Status", + ), + ), + ("message", models.CharField(blank=True, max_length=1024)), + ( + "video", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="transcoding", + to="pipeline.Video", + ), + ), ], ), migrations.CreateModel( - name='VideoUploadUrl', + name="VideoUploadUrl", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('public_video_id', models.CharField(default=pipeline.utils.generate_random_id, max_length=20, null=True, unique=True, validators=[django.core.validators.MinLengthValidator(1)])), - ('filename', models.CharField(max_length=128, verbose_name='Uploaded file name')), - ('expires_at', models.IntegerField(db_index=True, verbose_name='Timestamp at which the url expires')), - ('was_used', models.BooleanField(db_index=True, default=False, verbose_name='Was the upload url used?')), - ('last_checked', models.DateTimeField(blank=True, db_index=True, null=True, verbose_name='Last time it was checked if the url was used')), - ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='video_upload_urls', to=settings.AUTH_USER_MODEL)), - ('playlist', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='pipeline.Playlist', verbose_name='Playlist to which the video will be added after upload')), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "public_video_id", + models.CharField( + default=pipeline.utils.generate_random_id, + max_length=20, + null=True, + unique=True, + validators=[django.core.validators.MinLengthValidator(1)], + ), + ), + ( + "filename", + models.CharField(max_length=128, verbose_name="Uploaded file name"), + ), + ( + "expires_at", + models.IntegerField( + db_index=True, verbose_name="Timestamp at which the url expires" + ), + ), + ( + "was_used", + models.BooleanField( + db_index=True, + default=False, + verbose_name="Was the upload url used?", + ), + ), + ( + "last_checked", + models.DateTimeField( + blank=True, + db_index=True, + null=True, + verbose_name="Last time it was checked if the url was used", + ), + ), + ( + "owner", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="video_upload_urls", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "playlist", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="pipeline.Playlist", + verbose_name="Playlist to which the video will be added after upload", + ), + ), ], ), migrations.AddField( - model_name='playlist', - name='videos', - field=models.ManyToManyField(related_name='playlists', to='pipeline.Video'), + model_name="playlist", + name="videos", + field=models.ManyToManyField(related_name="playlists", to="pipeline.Video"), ), ] diff --git a/pipeline/migrations/0002_auto_20160824_0640.py b/pipeline/migrations/0002_auto_20160824_0640.py index 3930f40..71c7ebf 100644 --- a/pipeline/migrations/0002_auto_20160824_0640.py +++ b/pipeline/migrations/0002_auto_20160824_0640.py @@ -7,13 +7,8 @@ class Migration(migrations.Migration): - dependencies = [ - ('pipeline', '0001_initial'), - ] + dependencies = [("pipeline", "0001_initial")] operations = [ - migrations.RenameModel( - old_name='VideoSubtitles', - new_name='Subtitles', - ), + migrations.RenameModel(old_name="VideoSubtitles", new_name="Subtitles") ] diff --git a/pipeline/migrations/0003_auto_20160824_0733.py b/pipeline/migrations/0003_auto_20160824_0733.py index c9fc27e..c3012bd 100644 --- a/pipeline/migrations/0003_auto_20160824_0733.py +++ b/pipeline/migrations/0003_auto_20160824_0733.py @@ -7,13 +7,6 @@ class Migration(migrations.Migration): - dependencies = [ - ('pipeline', '0002_auto_20160824_0640'), - ] + dependencies = [("pipeline", "0002_auto_20160824_0640")] - operations = [ - migrations.RenameModel( - old_name='Subtitles', - new_name='Subtitle', - ), - ] + operations = [migrations.RenameModel(old_name="Subtitles", new_name="Subtitle")] diff --git a/pipeline/migrations/0004_auto_20160825_0741.py b/pipeline/migrations/0004_auto_20160825_0741.py index 740328d..d6a2727 100644 --- a/pipeline/migrations/0004_auto_20160825_0741.py +++ b/pipeline/migrations/0004_auto_20160825_0741.py @@ -7,14 +7,22 @@ class Migration(migrations.Migration): - dependencies = [ - ('pipeline', '0003_auto_20160824_0733'), - ] + dependencies = [("pipeline", "0003_auto_20160824_0733")] operations = [ migrations.AlterField( - model_name='videotranscoding', - name='status', - field=models.CharField(choices=[('pending', 'Pending'), ('processing', 'Processing'), ('failed', 'Failed'), ('success', 'Success')], default='pending', max_length=32, verbose_name='Status'), - ), + model_name="videotranscoding", + name="status", + field=models.CharField( + choices=[ + ("pending", "Pending"), + ("processing", "Processing"), + ("failed", "Failed"), + ("success", "Success"), + ], + default="pending", + max_length=32, + verbose_name="Status", + ), + ) ] diff --git a/pipeline/migrations/0005_auto_20160825_0819.py b/pipeline/migrations/0005_auto_20160825_0819.py index 47798e7..9ff4690 100644 --- a/pipeline/migrations/0005_auto_20160825_0819.py +++ b/pipeline/migrations/0005_auto_20160825_0819.py @@ -8,23 +8,24 @@ class Migration(migrations.Migration): - dependencies = [ - ('pipeline', '0004_auto_20160825_0741'), - ] + dependencies = [("pipeline", "0004_auto_20160825_0741")] operations = [ - migrations.RenameModel( - old_name='VideoTranscoding', - new_name='ProcessingState', - ), + migrations.RenameModel(old_name="VideoTranscoding", new_name="ProcessingState"), migrations.AlterField( - model_name='processingstate', - name='started_at', - field=models.DateTimeField(auto_now=True, verbose_name='Time of processing job start'), + model_name="processingstate", + name="started_at", + field=models.DateTimeField( + auto_now=True, verbose_name="Time of processing job start" + ), ), migrations.AlterField( - model_name='processingstate', - name='video', - field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='processing_state', to='pipeline.Video'), + model_name="processingstate", + name="video", + field=models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="processing_state", + to="pipeline.Video", + ), ), ] diff --git a/pipeline/migrations/0006_auto_20160905_1245.py b/pipeline/migrations/0006_auto_20160905_1245.py index 5ec8ed8..e181b09 100644 --- a/pipeline/migrations/0006_auto_20160905_1245.py +++ b/pipeline/migrations/0006_auto_20160905_1245.py @@ -7,14 +7,23 @@ class Migration(migrations.Migration): - dependencies = [ - ('pipeline', '0005_auto_20160825_0819'), - ] + dependencies = [("pipeline", "0005_auto_20160825_0819")] operations = [ migrations.AlterField( - model_name='processingstate', - name='status', - field=models.CharField(choices=[('pending', 'Pending'), ('processing', 'Processing'), ('failed', 'Failed'), ('success', 'Success'), ('restart', 'Restart')], default='pending', max_length=32, verbose_name='Status'), - ), + model_name="processingstate", + name="status", + field=models.CharField( + choices=[ + ("pending", "Pending"), + ("processing", "Processing"), + ("failed", "Failed"), + ("success", "Success"), + ("restart", "Restart"), + ], + default="pending", + max_length=32, + verbose_name="Status", + ), + ) ] diff --git a/pipeline/migrations/0007_auto_20160913_1247.py b/pipeline/migrations/0007_auto_20160913_1247.py index 8108b98..2364c48 100644 --- a/pipeline/migrations/0007_auto_20160913_1247.py +++ b/pipeline/migrations/0007_auto_20160913_1247.py @@ -7,17 +7,9 @@ class Migration(migrations.Migration): - dependencies = [ - ('pipeline', '0006_auto_20160905_1245'), - ] + dependencies = [("pipeline", "0006_auto_20160905_1245")] operations = [ - migrations.RemoveField( - model_name='videouploadurl', - name='filename', - ), - migrations.RemoveField( - model_name='videouploadurl', - name='last_checked', - ), + migrations.RemoveField(model_name="videouploadurl", name="filename"), + migrations.RemoveField(model_name="videouploadurl", name="last_checked"), ] diff --git a/pipeline/migrations/0008_videouploadurl_origin.py b/pipeline/migrations/0008_videouploadurl_origin.py index f3d3c0b..64b2d99 100644 --- a/pipeline/migrations/0008_videouploadurl_origin.py +++ b/pipeline/migrations/0008_videouploadurl_origin.py @@ -7,14 +7,17 @@ class Migration(migrations.Migration): - dependencies = [ - ('pipeline', '0007_auto_20160913_1247'), - ] + dependencies = [("pipeline", "0007_auto_20160913_1247")] operations = [ migrations.AddField( - model_name='videouploadurl', - name='origin', - field=models.CharField(blank=True, max_length=256, null=True, verbose_name='Access-Control-Allow-Origin header value to add to CORS responses'), - ), + model_name="videouploadurl", + name="origin", + field=models.CharField( + blank=True, + max_length=256, + null=True, + verbose_name="Access-Control-Allow-Origin header value to add to CORS responses", + ), + ) ] diff --git a/pipeline/migrations/0009_auto_20160914_1204.py b/pipeline/migrations/0009_auto_20160914_1204.py index 2d25d02..d04da66 100644 --- a/pipeline/migrations/0009_auto_20160914_1204.py +++ b/pipeline/migrations/0009_auto_20160914_1204.py @@ -8,14 +8,91 @@ class Migration(migrations.Migration): - dependencies = [ - ('pipeline', '0008_videouploadurl_origin'), - ] + dependencies = [("pipeline", "0008_videouploadurl_origin")] operations = [ migrations.AlterField( - model_name='subtitle', - name='language', - field=models.CharField(choices=[('af', 'Afrikaans'), ('ar', 'Arabic'), ('az', 'Azerbaijani'), ('bg', 'Bulgarian'), ('be', 'Belarusian'), ('bn', 'Bengali'), ('br', 'Breton'), ('bs', 'Bosnian'), ('ca', 'Catalan'), ('cs', 'Czech'), ('cy', 'Welsh'), ('da', 'Danish'), ('de', 'German'), ('el', 'Greek'), ('en', 'English'), ('eo', 'Esperanto'), ('es', 'Spanish'), ('et', 'Estonian'), ('eu', 'Basque'), ('fa', 'Persian'), ('fi', 'Finnish'), ('fr', 'French'), ('fy', 'Frisian'), ('ga', 'Irish'), ('gd', 'Scottish Gaelic'), ('gl', 'Galician'), ('he', 'Hebrew'), ('hi', 'Hindi'), ('hr', 'Croatian'), ('hu', 'Hungarian'), ('ia', 'Interlingua'), ('id', 'Indonesian'), ('io', 'Ido'), ('is', 'Icelandic'), ('it', 'Italian'), ('ja', 'Japanese'), ('ka', 'Georgian'), ('kk', 'Kazakh'), ('km', 'Khmer'), ('kn', 'Kannada'), ('ko', 'Korean'), ('lb', 'Luxembourgish'), ('lt', 'Lithuanian'), ('lv', 'Latvian'), ('mk', 'Macedonian'), ('ml', 'Malayalam'), ('mn', 'Mongolian'), ('mr', 'Marathi'), ('my', 'Burmese'), ('nb', 'Norwegian Bokmål'), ('ne', 'Nepali'), ('nl', 'Dutch'), ('nn', 'Norwegian Nynorsk'), ('os', 'Ossetic'), ('pa', 'Punjabi'), ('pl', 'Polish'), ('pt', 'Portuguese'), ('ro', 'Romanian'), ('ru', 'Russian'), ('sk', 'Slovak'), ('sl', 'Slovenian'), ('sq', 'Albanian'), ('sr', 'Serbian'), ('sv', 'Swedish'), ('sw', 'Swahili'), ('ta', 'Tamil'), ('te', 'Telugu'), ('th', 'Thai'), ('tr', 'Turkish'), ('tt', 'Tatar'), ('uk', 'Ukrainian'), ('ur', 'Urdu'), ('vi', 'Vietnamese')], max_length=2, null=True, validators=[django.core.validators.MinLengthValidator(2)]), - ), + model_name="subtitle", + name="language", + field=models.CharField( + choices=[ + ("af", "Afrikaans"), + ("ar", "Arabic"), + ("az", "Azerbaijani"), + ("bg", "Bulgarian"), + ("be", "Belarusian"), + ("bn", "Bengali"), + ("br", "Breton"), + ("bs", "Bosnian"), + ("ca", "Catalan"), + ("cs", "Czech"), + ("cy", "Welsh"), + ("da", "Danish"), + ("de", "German"), + ("el", "Greek"), + ("en", "English"), + ("eo", "Esperanto"), + ("es", "Spanish"), + ("et", "Estonian"), + ("eu", "Basque"), + ("fa", "Persian"), + ("fi", "Finnish"), + ("fr", "French"), + ("fy", "Frisian"), + ("ga", "Irish"), + ("gd", "Scottish Gaelic"), + ("gl", "Galician"), + ("he", "Hebrew"), + ("hi", "Hindi"), + ("hr", "Croatian"), + ("hu", "Hungarian"), + ("ia", "Interlingua"), + ("id", "Indonesian"), + ("io", "Ido"), + ("is", "Icelandic"), + ("it", "Italian"), + ("ja", "Japanese"), + ("ka", "Georgian"), + ("kk", "Kazakh"), + ("km", "Khmer"), + ("kn", "Kannada"), + ("ko", "Korean"), + ("lb", "Luxembourgish"), + ("lt", "Lithuanian"), + ("lv", "Latvian"), + ("mk", "Macedonian"), + ("ml", "Malayalam"), + ("mn", "Mongolian"), + ("mr", "Marathi"), + ("my", "Burmese"), + ("nb", "Norwegian Bokmål"), + ("ne", "Nepali"), + ("nl", "Dutch"), + ("nn", "Norwegian Nynorsk"), + ("os", "Ossetic"), + ("pa", "Punjabi"), + ("pl", "Polish"), + ("pt", "Portuguese"), + ("ro", "Romanian"), + ("ru", "Russian"), + ("sk", "Slovak"), + ("sl", "Slovenian"), + ("sq", "Albanian"), + ("sr", "Serbian"), + ("sv", "Swedish"), + ("sw", "Swahili"), + ("ta", "Tamil"), + ("te", "Telugu"), + ("th", "Thai"), + ("tr", "Turkish"), + ("tt", "Tatar"), + ("uk", "Ukrainian"), + ("ur", "Urdu"), + ("vi", "Vietnamese"), + ], + max_length=2, + null=True, + validators=[django.core.validators.MinLengthValidator(2)], + ), + ) ] diff --git a/pipeline/migrations/0010_video_public_thumbnail_id.py b/pipeline/migrations/0010_video_public_thumbnail_id.py index 3867f97..6b479b2 100644 --- a/pipeline/migrations/0010_video_public_thumbnail_id.py +++ b/pipeline/migrations/0010_video_public_thumbnail_id.py @@ -4,19 +4,22 @@ import django.core.validators from django.db import migrations, models + import pipeline.utils class Migration(migrations.Migration): - dependencies = [ - ('pipeline', '0009_auto_20160914_1204'), - ] + dependencies = [("pipeline", "0009_auto_20160914_1204")] operations = [ migrations.AddField( - model_name='video', - name='public_thumbnail_id', - field=models.CharField(default=pipeline.utils.generate_long_random_id, max_length=20, validators=[django.core.validators.MinLengthValidator(20)]), - ), + model_name="video", + name="public_thumbnail_id", + field=models.CharField( + default=pipeline.utils.generate_long_random_id, + max_length=20, + validators=[django.core.validators.MinLengthValidator(20)], + ), + ) ] diff --git a/pipeline/migrations/0011_create_thumbnails.py b/pipeline/migrations/0011_create_thumbnails.py index b2664a7..27c10ea 100644 --- a/pipeline/migrations/0011_create_thumbnails.py +++ b/pipeline/migrations/0011_create_thumbnails.py @@ -21,6 +21,7 @@ def create_thumbnails(apps, schema_editor): except: pass + def delete_thumbnails(apps, schema_editor): Video = apps.get_model("pipeline", "Video") backend = pipeline.backend.get() @@ -33,10 +34,8 @@ def delete_thumbnails(apps, schema_editor): class Migration(migrations.Migration): - dependencies = [ - ('pipeline', '0010_video_public_thumbnail_id'), - ] + dependencies = [("pipeline", "0010_video_public_thumbnail_id")] operations = [ - migrations.RunPython(create_thumbnails, reverse_code=delete_thumbnails), + migrations.RunPython(create_thumbnails, reverse_code=delete_thumbnails) ] diff --git a/pipeline/migrations/0012_auto_20160915_1003.py b/pipeline/migrations/0012_auto_20160915_1003.py index 4b4bba8..62048f8 100644 --- a/pipeline/migrations/0012_auto_20160915_1003.py +++ b/pipeline/migrations/0012_auto_20160915_1003.py @@ -4,19 +4,23 @@ import django.core.validators from django.db import migrations, models + import pipeline.utils class Migration(migrations.Migration): - dependencies = [ - ('pipeline', '0011_create_thumbnails'), - ] + dependencies = [("pipeline", "0011_create_thumbnails")] operations = [ migrations.AlterField( - model_name='video', - name='public_thumbnail_id', - field=models.CharField(default=pipeline.utils.generate_long_random_id, max_length=20, unique=True, validators=[django.core.validators.MinLengthValidator(20)]), - ), + model_name="video", + name="public_thumbnail_id", + field=models.CharField( + default=pipeline.utils.generate_long_random_id, + max_length=20, + unique=True, + validators=[django.core.validators.MinLengthValidator(20)], + ), + ) ] diff --git a/pipeline/migrations/0013_auto_20180124_0930.py b/pipeline/migrations/0013_auto_20180124_0930.py index ed2c7d6..3ee87ba 100644 --- a/pipeline/migrations/0013_auto_20180124_0930.py +++ b/pipeline/migrations/0013_auto_20180124_0930.py @@ -8,14 +8,106 @@ class Migration(migrations.Migration): - dependencies = [ - ('pipeline', '0012_auto_20160915_1003'), - ] + dependencies = [("pipeline", "0012_auto_20160915_1003")] operations = [ migrations.AlterField( - model_name='subtitle', - name='language', - field=models.CharField(choices=[('af', 'Afrikaans'), ('ar', 'Arabic'), ('ast', 'Asturian'), ('az', 'Azerbaijani'), ('bg', 'Bulgarian'), ('be', 'Belarusian'), ('bn', 'Bengali'), ('br', 'Breton'), ('bs', 'Bosnian'), ('ca', 'Catalan'), ('cs', 'Czech'), ('cy', 'Welsh'), ('da', 'Danish'), ('de', 'German'), ('dsb', 'Lower Sorbian'), ('el', 'Greek'), ('en', 'English'), ('en-au', 'Australian English'), ('en-gb', 'British English'), ('eo', 'Esperanto'), ('es', 'Spanish'), ('es-ar', 'Argentinian Spanish'), ('es-co', 'Colombian Spanish'), ('es-mx', 'Mexican Spanish'), ('es-ni', 'Nicaraguan Spanish'), ('es-ve', 'Venezuelan Spanish'), ('et', 'Estonian'), ('eu', 'Basque'), ('fa', 'Persian'), ('fi', 'Finnish'), ('fr', 'French'), ('fy', 'Frisian'), ('ga', 'Irish'), ('gd', 'Scottish Gaelic'), ('gl', 'Galician'), ('he', 'Hebrew'), ('hi', 'Hindi'), ('hr', 'Croatian'), ('hsb', 'Upper Sorbian'), ('hu', 'Hungarian'), ('ia', 'Interlingua'), ('id', 'Indonesian'), ('io', 'Ido'), ('is', 'Icelandic'), ('it', 'Italian'), ('ja', 'Japanese'), ('ka', 'Georgian'), ('kk', 'Kazakh'), ('km', 'Khmer'), ('kn', 'Kannada'), ('ko', 'Korean'), ('lb', 'Luxembourgish'), ('lt', 'Lithuanian'), ('lv', 'Latvian'), ('mk', 'Macedonian'), ('ml', 'Malayalam'), ('mn', 'Mongolian'), ('mr', 'Marathi'), ('my', 'Burmese'), ('nb', 'Norwegian Bokmål'), ('ne', 'Nepali'), ('nl', 'Dutch'), ('nn', 'Norwegian Nynorsk'), ('os', 'Ossetic'), ('pa', 'Punjabi'), ('pl', 'Polish'), ('pt', 'Portuguese'), ('pt-br', 'Brazilian Portuguese'), ('ro', 'Romanian'), ('ru', 'Russian'), ('sk', 'Slovak'), ('sl', 'Slovenian'), ('sq', 'Albanian'), ('sr', 'Serbian'), ('sr-latn', 'Serbian Latin'), ('sv', 'Swedish'), ('sw', 'Swahili'), ('ta', 'Tamil'), ('te', 'Telugu'), ('th', 'Thai'), ('tr', 'Turkish'), ('tt', 'Tatar'), ('udm', 'Udmurt'), ('uk', 'Ukrainian'), ('ur', 'Urdu'), ('vi', 'Vietnamese'), ('zh-hans', 'Simplified Chinese'), ('zh-hant', 'Traditional Chinese')], max_length=7, null=True, validators=[django.core.validators.MinLengthValidator(2)]), - ), + model_name="subtitle", + name="language", + field=models.CharField( + choices=[ + ("af", "Afrikaans"), + ("ar", "Arabic"), + ("ast", "Asturian"), + ("az", "Azerbaijani"), + ("bg", "Bulgarian"), + ("be", "Belarusian"), + ("bn", "Bengali"), + ("br", "Breton"), + ("bs", "Bosnian"), + ("ca", "Catalan"), + ("cs", "Czech"), + ("cy", "Welsh"), + ("da", "Danish"), + ("de", "German"), + ("dsb", "Lower Sorbian"), + ("el", "Greek"), + ("en", "English"), + ("en-au", "Australian English"), + ("en-gb", "British English"), + ("eo", "Esperanto"), + ("es", "Spanish"), + ("es-ar", "Argentinian Spanish"), + ("es-co", "Colombian Spanish"), + ("es-mx", "Mexican Spanish"), + ("es-ni", "Nicaraguan Spanish"), + ("es-ve", "Venezuelan Spanish"), + ("et", "Estonian"), + ("eu", "Basque"), + ("fa", "Persian"), + ("fi", "Finnish"), + ("fr", "French"), + ("fy", "Frisian"), + ("ga", "Irish"), + ("gd", "Scottish Gaelic"), + ("gl", "Galician"), + ("he", "Hebrew"), + ("hi", "Hindi"), + ("hr", "Croatian"), + ("hsb", "Upper Sorbian"), + ("hu", "Hungarian"), + ("ia", "Interlingua"), + ("id", "Indonesian"), + ("io", "Ido"), + ("is", "Icelandic"), + ("it", "Italian"), + ("ja", "Japanese"), + ("ka", "Georgian"), + ("kk", "Kazakh"), + ("km", "Khmer"), + ("kn", "Kannada"), + ("ko", "Korean"), + ("lb", "Luxembourgish"), + ("lt", "Lithuanian"), + ("lv", "Latvian"), + ("mk", "Macedonian"), + ("ml", "Malayalam"), + ("mn", "Mongolian"), + ("mr", "Marathi"), + ("my", "Burmese"), + ("nb", "Norwegian Bokmål"), + ("ne", "Nepali"), + ("nl", "Dutch"), + ("nn", "Norwegian Nynorsk"), + ("os", "Ossetic"), + ("pa", "Punjabi"), + ("pl", "Polish"), + ("pt", "Portuguese"), + ("pt-br", "Brazilian Portuguese"), + ("ro", "Romanian"), + ("ru", "Russian"), + ("sk", "Slovak"), + ("sl", "Slovenian"), + ("sq", "Albanian"), + ("sr", "Serbian"), + ("sr-latn", "Serbian Latin"), + ("sv", "Swedish"), + ("sw", "Swahili"), + ("ta", "Tamil"), + ("te", "Telugu"), + ("th", "Thai"), + ("tr", "Turkish"), + ("tt", "Tatar"), + ("udm", "Udmurt"), + ("uk", "Ukrainian"), + ("ur", "Urdu"), + ("vi", "Vietnamese"), + ("zh-hans", "Simplified Chinese"), + ("zh-hant", "Traditional Chinese"), + ], + max_length=7, + null=True, + validators=[django.core.validators.MinLengthValidator(2)], + ), + ) ] diff --git a/pipeline/models.py b/pipeline/models.py index 9db9296..25317ce 100644 --- a/pipeline/models.py +++ b/pipeline/models.py @@ -1,35 +1,40 @@ from django.conf import global_settings from django.contrib.auth.models import User -from django.core.validators import MinLengthValidator, MinValueValidator, MaxValueValidator +from django.core.validators import ( + MaxValueValidator, + MinLengthValidator, + MinValueValidator, +) from django.db import models -from django.db.models.signals import post_save, post_delete +from django.db.models.signals import post_delete, post_save from django.dispatch import receiver -from . import backend -from . import cache -from . import managers -from . import utils +from . import backend, cache, managers, utils class Video(models.Model): title = models.CharField(max_length=100) public_id = models.CharField( - max_length=20, unique=True, + max_length=20, + unique=True, validators=[MinLengthValidator(1)], - blank=False, null=True, + blank=False, + null=True, default=utils.generate_random_id, ) public_thumbnail_id = models.CharField( - max_length=20, unique=True, + max_length=20, + unique=True, validators=[MinLengthValidator(20)], - blank=False, null=False, + blank=False, + null=False, default=utils.generate_long_random_id, ) owner = models.ForeignKey(User, on_delete=models.CASCADE) def __str__(self): - return '{} - {}'.format(self.public_id, self.title) + return "{} - {}".format(self.public_id, self.title) @property def processing_status(self): @@ -59,17 +64,19 @@ def create_video_processing_state(sender, instance=None, created=False, **kwargs class Playlist(models.Model): name = models.CharField(max_length=128, db_index=True) - videos = models.ManyToManyField(Video, related_name='playlists') + videos = models.ManyToManyField(Video, related_name="playlists") owner = models.ForeignKey(User, on_delete=models.CASCADE) public_id = models.CharField( - max_length=20, unique=True, + max_length=20, + unique=True, validators=[MinLengthValidator(1)], - blank=False, null=True, + blank=False, + null=True, default=utils.generate_random_id, ) def __str__(self): - return '{} - {}'.format(self.public_id, self.name) + return "{} - {}".format(self.public_id, self.name) class VideoUploadUrl(models.Model): @@ -79,36 +86,36 @@ class VideoUploadUrl(models.Model): that an upload that has started just before the expiry date should proceed normally. """ + public_video_id = models.CharField( - max_length=20, unique=True, + max_length=20, + unique=True, validators=[MinLengthValidator(1)], - blank=False, null=True, + blank=False, + null=True, default=utils.generate_random_id, ) expires_at = models.IntegerField( - verbose_name="Timestamp at which the url expires", - db_index=True + verbose_name="Timestamp at which the url expires", db_index=True ) was_used = models.BooleanField( - verbose_name="Was the upload url used?", - default=False, - db_index=True + verbose_name="Was the upload url used?", default=False, db_index=True ) owner = models.ForeignKey( - User, - related_name='video_upload_urls', - on_delete=models.CASCADE + User, related_name="video_upload_urls", on_delete=models.CASCADE ) playlist = models.ForeignKey( Playlist, verbose_name="Playlist to which the video will be added after upload", on_delete=models.CASCADE, - blank=True, null=True + blank=True, + null=True, ) origin = models.CharField( verbose_name="Access-Control-Allow-Origin header value to add to CORS responses", max_length=256, - blank=True, null=True + blank=True, + null=True, ) objects = managers.VideoUploadUrlManager() @@ -119,32 +126,29 @@ def __str__(self): class ProcessingState(models.Model): - STATUS_PENDING = 'pending' - STATUS_PROCESSING = 'processing' - STATUS_FAILED = 'failed' - STATUS_SUCCESS = 'success' - STATUS_RESTART = 'restart' + STATUS_PENDING = "pending" + STATUS_PROCESSING = "processing" + STATUS_FAILED = "failed" + STATUS_SUCCESS = "success" + STATUS_RESTART = "restart" STATUSES = ( - (STATUS_PENDING, 'Pending'), - (STATUS_PROCESSING, 'Processing'), - (STATUS_FAILED, 'Failed'), - (STATUS_SUCCESS, 'Success'), - (STATUS_RESTART, 'Restart'), + (STATUS_PENDING, "Pending"), + (STATUS_PROCESSING, "Processing"), + (STATUS_FAILED, "Failed"), + (STATUS_SUCCESS, "Success"), + (STATUS_RESTART, "Restart"), ) video = models.OneToOneField( - Video, - related_name='processing_state', - on_delete=models.CASCADE + Video, related_name="processing_state", on_delete=models.CASCADE ) started_at = models.DateTimeField( - verbose_name="Time of processing job start", - auto_now=True + verbose_name="Time of processing job start", auto_now=True ) progress = models.FloatField( verbose_name="Progress percentage", default=0, - validators=[MinValueValidator(0), MaxValueValidator(100)] + validators=[MinValueValidator(0), MaxValueValidator(100)], ) status = models.CharField( verbose_name="Status", @@ -156,20 +160,18 @@ class ProcessingState(models.Model): message = models.CharField(max_length=1024, blank=True) def __str__(self): - return '{} - {}'.format(self.video, self.status) + return "{} - {}".format(self.video, self.status) class Subtitle(models.Model): - video = models.ForeignKey( - Video, - related_name='subtitles', - on_delete=models.CASCADE - ) + video = models.ForeignKey(Video, related_name="subtitles", on_delete=models.CASCADE) public_id = models.CharField( - max_length=20, unique=True, + max_length=20, + unique=True, validators=[MinLengthValidator(1)], - blank=False, null=True, + blank=False, + null=True, default=utils.generate_random_id, ) language = models.CharField( @@ -177,7 +179,7 @@ class Subtitle(models.Model): validators=[MinLengthValidator(2)], choices=global_settings.LANGUAGES, null=True, - blank=False + blank=False, ) @property @@ -187,16 +189,12 @@ def url(self): ) def __str__(self): - return '{} - {} [{}]'.format(self.public_id, self.video, self.language) + return "{} - {} [{}]".format(self.public_id, self.video, self.language) class VideoFormat(models.Model): - video = models.ForeignKey( - Video, - related_name='formats', - on_delete=models.CASCADE - ) + video = models.ForeignKey(Video, related_name="formats", on_delete=models.CASCADE) name = models.CharField(max_length=128) bitrate = models.FloatField(validators=[MinValueValidator(0)]) @@ -205,7 +203,7 @@ def url(self): return backend.get().video_url(self.video.public_id, self.name) def __str__(self): - return '{} - {} [{}]'.format(self.name, self.video, self.bitrate) + return "{} - {} [{}]".format(self.name, self.video, self.bitrate) @receiver([post_save, post_delete], sender=Video) @@ -213,6 +211,7 @@ def invalidate_video_cache(sender, instance=None, created=False, **kwargs): if instance: invalidate_cache(instance.public_id) + @receiver([post_save, post_delete], sender=Subtitle) @receiver([post_save, post_delete], sender=ProcessingState) @receiver([post_save, post_delete], sender=VideoFormat) @@ -223,5 +222,6 @@ def invalidate_related_video_cache(sender, instance=None, created=False, **kwarg if instance: invalidate_cache(instance.video.public_id) + def invalidate_cache(public_video_id): cache.invalidate(public_video_id) diff --git a/pipeline/tasks.py b/pipeline/tasks.py index 242f41d..96035a3 100644 --- a/pipeline/tasks.py +++ b/pipeline/tasks.py @@ -3,16 +3,15 @@ from time import sleep from celery import shared_task -from django.db.transaction import TransactionManagementError +import pycaption + from django.core.cache import cache +from django.db.transaction import TransactionManagementError from django.utils.timezone import now -import pycaption from videofront.celery_videofront import send_task -from . import backend -from . import exceptions -from . import models -from . import utils + +from . import backend, exceptions, models, utils logger = logging.getLogger(__name__) @@ -68,6 +67,7 @@ def acquire_lock(name, expires_in=None): return True raise exceptions.LockUnavailable(name) + def release_lock(name): """ Release a lock for all. Note that the lock will be released even if it was @@ -83,6 +83,7 @@ def release_lock(name): except TransactionManagementError: logger.error("Could not release lock %s", name) + def upload_video(public_video_id, file_object): """ Store a video file for transcoding. @@ -92,9 +93,13 @@ def upload_video(public_video_id, file_object): file_object (file) """ # Make upload url unavailable immediately to avoid race conditions - models.VideoUploadUrl.objects.filter(public_video_id=public_video_id).update(was_used=True) + models.VideoUploadUrl.objects.filter(public_video_id=public_video_id).update( + was_used=True + ) - video_upload_url = models.VideoUploadUrl.objects.get(public_video_id=public_video_id) + video_upload_url = models.VideoUploadUrl.objects.get( + public_video_id=public_video_id + ) # Upload video backend.get().upload_video(public_video_id, file_object) @@ -103,29 +108,37 @@ def upload_video(public_video_id, file_object): video = models.Video.objects.create( public_id=video_upload_url.public_video_id, owner=video_upload_url.owner, - title=file_object.name + title=file_object.name, ) if video_upload_url.playlist: video.playlists.add(video_upload_url.playlist) # Start transcoding - send_task('transcode_video', args=(public_video_id,)) + send_task("transcode_video", args=(public_video_id,)) + -@shared_task(name='transcode_video_restart') +@shared_task(name="transcode_video_restart") def transcode_video_restart(): - with Lock('TASK_LOCK_TRANSCODE_VIDEO_RESTART', 60) as lock: + with Lock("TASK_LOCK_TRANSCODE_VIDEO_RESTART", 60) as lock: if lock.is_acquired: - for processing_state in models.ProcessingState.objects.filter(status=models.ProcessingState.STATUS_RESTART): - send_task('transcode_video', args=(processing_state.video.public_id,), kwargs={'delete': False}) + for processing_state in models.ProcessingState.objects.filter( + status=models.ProcessingState.STATUS_RESTART + ): + send_task( + "transcode_video", + args=(processing_state.video.public_id,), + kwargs={"delete": False}, + ) + -@shared_task(name='transcode_video') +@shared_task(name="transcode_video") def transcode_video(public_video_id, delete=True): """ Args: public_video_id (str) delete (bool): delete video on failure """ - with Lock('TASK_LOCK_TRANSCODE_VIDEO:' + public_video_id, 3600) as lock: + with Lock("TASK_LOCK_TRANSCODE_VIDEO:" + public_video_id, 3600) as lock: if lock.is_acquired: try: models.invalidate_cache(public_video_id) @@ -135,24 +148,22 @@ def transcode_video(public_video_id, delete=True): message = "\n".join([str(arg) for arg in e.args]) models.ProcessingState.objects.filter( video__public_id=public_video_id - ).update( - status=models.ProcessingState.STATUS_FAILED, - message=message, - ) + ).update(status=models.ProcessingState.STATUS_FAILED, message=message) raise finally: models.invalidate_cache(public_video_id) + def _transcode_video(public_video_id, delete=True): """ This function is not thread-safe. It should only be called by the transcode_video task. """ video = models.Video.objects.get(public_id=public_video_id) - processing_state = models.ProcessingState.objects.filter(video__public_id=public_video_id) + processing_state = models.ProcessingState.objects.filter( + video__public_id=public_video_id + ) processing_state.update( - progress=0, - status=models.ProcessingState.STATUS_PENDING, - started_at=now() + progress=0, status=models.ProcessingState.STATUS_PENDING, started_at=now() ) jobs = backend.get().start_transcoding(public_video_id) @@ -162,9 +173,14 @@ def _transcode_video(public_video_id, delete=True): jobs_progress = [0] * len(jobs) while len(success_job_indexes) + len(error_job_indexes) < len(jobs): for job_index, job in enumerate(jobs): - if job_index not in success_job_indexes and job_index not in error_job_indexes: + if ( + job_index not in success_job_indexes + and job_index not in error_job_indexes + ): try: - jobs_progress[job_index], finished = backend.get().check_progress(job) + jobs_progress[job_index], finished = backend.get().check_progress( + job + ) if finished: success_job_indexes.append(job_index) except exceptions.TranscodingFailed as e: @@ -177,7 +193,7 @@ def _transcode_video(public_video_id, delete=True): # the transcoding process. processing_state.update( progress=sum(jobs_progress) * 1. / len(jobs), - status=models.ProcessingState.STATUS_PROCESSING + status=models.ProcessingState.STATUS_PROCESSING, ) # Create thumbnail @@ -202,7 +218,9 @@ def _transcode_video(public_video_id, delete=True): # Create video formats first so that they are available as soon as the # video object becomes available from the API for format_name, bitrate in backend.get().iter_formats(public_video_id): - models.VideoFormat.objects.create(video=video, name=format_name, bitrate=bitrate) + models.VideoFormat.objects.create( + video=video, name=format_name, bitrate=bitrate + ) processing_state.update(status=models.ProcessingState.STATUS_SUCCESS) @@ -210,6 +228,7 @@ def _transcode_video(public_video_id, delete=True): if not models.Video.objects.filter(public_id=public_video_id).exists(): delete_video(public_video_id) + def upload_subtitle(public_video_id, subtitle_public_id, language_code, content): """ Convert subtitle to VTT and upload it. @@ -221,7 +240,7 @@ def upload_subtitle(public_video_id, subtitle_public_id, language_code, content) content (bytes) """ # Note: if this ever raises an exception, we should convert it to SubtitleInvalid - content = content.decode('utf-8') + content = content.decode("utf-8") # Convert to VTT, whatever the initial format content = content.strip("\ufeff\n\r") @@ -231,7 +250,10 @@ def upload_subtitle(public_video_id, subtitle_public_id, language_code, content) if sub_reader != pycaption.WebVTTReader: content = pycaption.WebVTTWriter().write(sub_reader().read(content)) - backend.get().upload_subtitle(public_video_id, subtitle_public_id, language_code, content) + backend.get().upload_subtitle( + public_video_id, subtitle_public_id, language_code, content + ) + def upload_thumbnail(public_video_id, file_object): """ @@ -242,7 +264,7 @@ def upload_thumbnail(public_video_id, file_object): content (bytes) """ video = models.Video.objects.get(public_id=public_video_id) - out_img = NamedTemporaryFile(mode='rb', suffix=".jpg") + out_img = NamedTemporaryFile(mode="rb", suffix=".jpg") try: utils.make_thumbnail(file_object, out_img.name) @@ -262,15 +284,18 @@ def upload_thumbnail(public_video_id, file_object): video.public_thumbnail_id = thumb_id video.save() + def delete_video(public_video_id): """ Delete all video assets """ backend.get().delete_video(public_video_id) + def delete_subtitle(public_video_id, public_subtitle_id): """ Delete subtitle associated to video""" backend.get().delete_subtitle(public_video_id, public_subtitle_id) -@shared_task(name='clean_upload_urls') + +@shared_task(name="clean_upload_urls") def clean_upload_urls(): """ Remove video upload urls which cannot be used anymore. diff --git a/pipeline/tests/factories.py b/pipeline/tests/factories.py index 9a6822a..c705289 100644 --- a/pipeline/tests/factories.py +++ b/pipeline/tests/factories.py @@ -1,11 +1,14 @@ -import factory.django from django.contrib.auth.models import User + +import factory.django + from pipeline import models class UserFactory(factory.django.DjangoModelFactory): class Meta: model = User + username = factory.Sequence(lambda n: "User %d" % n) diff --git a/pipeline/tests/test_backend.py b/pipeline/tests/test_backend.py index b7393a4..739f968 100644 --- a/pipeline/tests/test_backend.py +++ b/pipeline/tests/test_backend.py @@ -5,7 +5,6 @@ class PipelineBackendTests(TestCase): - @override_settings(PLUGIN_BACKEND=None) def test_undefined_backend(self): self.assertRaises(backend.UndefinedPluginBackend, backend.get) diff --git a/pipeline/tests/test_models.py b/pipeline/tests/test_models.py index 24a3dc9..145bd33 100644 --- a/pipeline/tests/test_models.py +++ b/pipeline/tests/test_models.py @@ -7,29 +7,25 @@ class VideoUploadUrlTests(TestCase): - def test_available(self): factories.VideoUploadUrlFactory( - public_video_id='available', - expires_at=time() + 3600 + public_video_id="available", expires_at=time() + 3600 ) factories.VideoUploadUrlFactory( - public_video_id='almost_expired', - expires_at=time() - 3000, + public_video_id="almost_expired", expires_at=time() - 3000 ) factories.VideoUploadUrlFactory( - public_video_id='used', - expires_at=time() + 3600, - was_used=True, + public_video_id="used", expires_at=time() + 3600, was_used=True ) factories.VideoUploadUrlFactory( - public_video_id='expired', - expires_at=time() - 3600, + public_video_id="expired", expires_at=time() - 3600 ) - available_video_ids = [u.public_video_id for u in models.VideoUploadUrl.objects.available()] + available_video_ids = [ + u.public_video_id for u in models.VideoUploadUrl.objects.available() + ] - self.assertIn('available', available_video_ids) - self.assertIn('almost_expired', available_video_ids) - self.assertNotIn('used', available_video_ids) - self.assertNotIn('expired', available_video_ids) + self.assertIn("available", available_video_ids) + self.assertIn("almost_expired", available_video_ids) + self.assertNotIn("used", available_video_ids) + self.assertNotIn("expired", available_video_ids) diff --git a/pipeline/tests/test_tasks.py b/pipeline/tests/test_tasks.py index e36ae47..ba1b729 100644 --- a/pipeline/tests/test_tasks.py +++ b/pipeline/tests/test_tasks.py @@ -7,9 +7,7 @@ from django.test.utils import override_settings from django.urls import reverse -from pipeline import exceptions -from pipeline import models -from pipeline import tasks +from pipeline import exceptions, models, tasks from pipeline.tests import factories from videofront.celery_videofront import send_task @@ -20,26 +18,26 @@ class LockTests(TransactionTestCase): """ def setUp(self): - tasks.release_lock('dummylock') + tasks.release_lock("dummylock") def tearDown(self): - tasks.release_lock('dummylock') + tasks.release_lock("dummylock") def test_acquire_release_lock_cycle(self): self.assertTrue(tasks.acquire_lock("dummylock")) self.assertRaises(exceptions.LockUnavailable, tasks.acquire_lock, "dummylock") - tasks.release_lock('dummylock') + tasks.release_lock("dummylock") self.assertTrue(tasks.acquire_lock("dummylock")) def test_release_lock_with_integrity_error(self): def failing_task(): - tasks.acquire_lock('dummylock', 3600) + tasks.acquire_lock("dummylock", 3600) try: models.Video.objects.create(public_id="id") models.Video.objects.create(public_id="id") finally: - tasks.release_lock('dummylock') + tasks.release_lock("dummylock") self.assertRaises(IntegrityError, failing_task) self.assertTrue(tasks.acquire_lock("dummylock")) @@ -47,37 +45,38 @@ def failing_task(): def test_context_manager(self): # 1) Lock is available - with tasks.Lock('dummylock') as lock: + with tasks.Lock("dummylock") as lock: self.assertTrue(lock.is_acquired) - self.assertRaises(exceptions.LockUnavailable, tasks.acquire_lock, "dummylock") + self.assertRaises( + exceptions.LockUnavailable, tasks.acquire_lock, "dummylock" + ) self.assertFalse(lock.is_acquired) # 2) Lock is unavailable tasks.acquire_lock("dummylock") - with tasks.Lock('dummylock') as lock: + with tasks.Lock("dummylock") as lock: self.assertFalse(lock.is_acquired) self.assertRaises(exceptions.LockUnavailable, tasks.acquire_lock, "dummylock") class TasksTests(TestCase): - def test_upload_video(self): - mock_backend = Mock(return_value=Mock( - upload_video=Mock(), - start_transcoding=Mock(return_value=[]), - iter_formats=Mock(return_value=[]), - )) + mock_backend = Mock( + return_value=Mock( + upload_video=Mock(), + start_transcoding=Mock(return_value=[]), + iter_formats=Mock(return_value=[]), + ) + ) factories.VideoUploadUrlFactory( - was_used=False, - public_video_id='videoid', - expires_at=time() + 3600 + was_used=False, public_video_id="videoid", expires_at=time() + 3600 ) file_object = Mock() file_object.name = "Some video.mp4" with override_settings(PLUGIN_BACKEND=mock_backend): - tasks.upload_video('videoid', file_object) + tasks.upload_video("videoid", file_object) self.assertEqual(1, models.Video.objects.count()) self.assertEqual(1, models.VideoUploadUrl.objects.count()) @@ -88,71 +87,78 @@ def test_upload_video(self): self.assertTrue(video_upload_url.was_used) def test_upload_url_invalidated_after_failed_upload(self): - mock_backend = Mock(return_value=Mock( - upload_video=Mock(side_effect=ValueError), - )) + mock_backend = Mock( + return_value=Mock(upload_video=Mock(side_effect=ValueError)) + ) factories.VideoUploadUrlFactory( - was_used=False, - public_video_id='videoid', - expires_at=time() + 3600 + was_used=False, public_video_id="videoid", expires_at=time() + 3600 ) file_object = Mock() file_object.name = "Some video.mp4" with override_settings(PLUGIN_BACKEND=mock_backend): - self.assertRaises(ValueError, tasks.upload_video, 'videoid', file_object) + self.assertRaises(ValueError, tasks.upload_video, "videoid", file_object) self.assertEqual(0, models.Video.objects.count()) self.assertEqual(0, models.VideoUploadUrl.objects.available().count()) - def test_transcode_video_success(self): - factories.VideoFactory(public_id='videoid', public_thumbnail_id='thumbid') - mock_backend = Mock(return_value=Mock( - start_transcoding=Mock(return_value=['job1']), - check_progress=Mock(return_value=(42, True)), - iter_formats=Mock(return_value=[('SD', 128)]), - create_thumbnail=Mock(), - )) + factories.VideoFactory(public_id="videoid", public_thumbnail_id="thumbid") + mock_backend = Mock( + return_value=Mock( + start_transcoding=Mock(return_value=["job1"]), + check_progress=Mock(return_value=(42, True)), + iter_formats=Mock(return_value=[("SD", 128)]), + create_thumbnail=Mock(), + ) + ) with override_settings(PLUGIN_BACKEND=mock_backend): - tasks.transcode_video('videoid') + tasks.transcode_video("videoid") self.assertEqual(1, models.ProcessingState.objects.count()) video_processing_state = models.ProcessingState.objects.get() - self.assertEqual(models.ProcessingState.STATUS_SUCCESS, video_processing_state.status) + self.assertEqual( + models.ProcessingState.STATUS_SUCCESS, video_processing_state.status + ) self.assertEqual("", video_processing_state.message) self.assertEqual(42, video_processing_state.progress) - mock_backend.return_value.create_thumbnail.assert_called_once_with('videoid', 'thumbid') - mock_backend.return_value.check_progress.assert_called_once_with('job1') + mock_backend.return_value.create_thumbnail.assert_called_once_with( + "videoid", "thumbid" + ) + mock_backend.return_value.check_progress.assert_called_once_with("job1") self.assertEqual(1, models.VideoFormat.objects.count()) video_format = models.VideoFormat.objects.get() - self.assertEqual('videoid', video_format.video.public_id) - self.assertEqual('SD', video_format.name) + self.assertEqual("videoid", video_format.video.public_id) + self.assertEqual("SD", video_format.name) self.assertEqual(128, video_format.bitrate) def test_transcode_video_failure(self): - factories.VideoFactory(public_id='videoid') + factories.VideoFactory(public_id="videoid") def check_progress(job): - if job == 'job1': + if job == "job1": # job1 finishes - raise exceptions.TranscodingFailed('error message') + raise exceptions.TranscodingFailed("error message") else: # job2 finishes return 100, True - mock_backend = Mock(return_value=Mock( - start_transcoding=Mock(return_value=['job1', 'job2']), - check_progress=check_progress, - iter_formats=Mock(return_value=[]), - )) + mock_backend = Mock( + return_value=Mock( + start_transcoding=Mock(return_value=["job1", "job2"]), + check_progress=check_progress, + iter_formats=Mock(return_value=[]), + ) + ) with override_settings(PLUGIN_BACKEND=mock_backend): - tasks.transcode_video('videoid') + tasks.transcode_video("videoid") self.assertEqual(1, models.ProcessingState.objects.count()) video_processing_state = models.ProcessingState.objects.get() - self.assertEqual(models.ProcessingState.STATUS_FAILED, video_processing_state.status) + self.assertEqual( + models.ProcessingState.STATUS_FAILED, video_processing_state.status + ) self.assertEqual("error message", video_processing_state.message) self.assertEqual(50, video_processing_state.progress) mock_backend.return_value.create_thumbnail.assert_not_called() @@ -164,121 +170,147 @@ def test_video_transcoding_failure_invalidates_cache(self): user.save() self.client.login(username="test", password="password") - factories.VideoFactory(public_id='videoid', owner=user) + factories.VideoFactory(public_id="videoid", owner=user) - mock_backend = Mock(return_value=Mock( - start_transcoding=Mock(return_value=['job']), - check_progress=Mock(side_effect=exceptions.TranscodingFailed), - )) + mock_backend = Mock( + return_value=Mock( + start_transcoding=Mock(return_value=["job"]), + check_progress=Mock(side_effect=exceptions.TranscodingFailed), + ) + ) - video_pre_transcoding = self.client.get(reverse("api:v1:video-detail", kwargs={"id": 'videoid'})).json() + video_pre_transcoding = self.client.get( + reverse("api:v1:video-detail", kwargs={"id": "videoid"}) + ).json() with override_settings(PLUGIN_BACKEND=mock_backend): - tasks.transcode_video('videoid') - video_post_transcoding = self.client.get(reverse("api:v1:video-detail", kwargs={"id": 'videoid'})).json() - - self.assertEqual('pending', video_pre_transcoding['processing']['status']) - self.assertEqual('failed', video_post_transcoding['processing']['status']) + tasks.transcode_video("videoid") + video_post_transcoding = self.client.get( + reverse("api:v1:video-detail", kwargs={"id": "videoid"}) + ).json() + self.assertEqual("pending", video_pre_transcoding["processing"]["status"]) + self.assertEqual("failed", video_post_transcoding["processing"]["status"]) def test_transcode_video_unexpected_failure(self): - factories.VideoFactory(public_id='videoid') + factories.VideoFactory(public_id="videoid") - mock_backend = Mock(return_value=Mock( - start_transcoding=Mock(side_effect=ValueError(666, "random error")) - )) + mock_backend = Mock( + return_value=Mock( + start_transcoding=Mock(side_effect=ValueError(666, "random error")) + ) + ) with override_settings(PLUGIN_BACKEND=mock_backend): - self.assertRaises(ValueError, tasks.transcode_video, 'videoid') + self.assertRaises(ValueError, tasks.transcode_video, "videoid") video_processing_state = models.ProcessingState.objects.get() - self.assertEqual(models.ProcessingState.STATUS_FAILED, video_processing_state.status) + self.assertEqual( + models.ProcessingState.STATUS_FAILED, video_processing_state.status + ) self.assertEqual("666\nrandom error", video_processing_state.message) def test_transcode_video_twice(self): - factories.VideoFactory(public_id='videoid') - mock_backend = Mock(return_value=Mock( - start_transcoding=Mock(return_value=['job1']), - iter_formats=Mock(return_value=[]), - )) + factories.VideoFactory(public_id="videoid") + mock_backend = Mock( + return_value=Mock( + start_transcoding=Mock(return_value=["job1"]), + iter_formats=Mock(return_value=[]), + ) + ) # First attempt: failure - mock_backend.return_value.check_progress = Mock(side_effect=exceptions.TranscodingFailed) + mock_backend.return_value.check_progress = Mock( + side_effect=exceptions.TranscodingFailed + ) with override_settings(PLUGIN_BACKEND=mock_backend): - tasks.transcode_video('videoid') + tasks.transcode_video("videoid") # Second attempt: success mock_backend.return_value.check_progress = Mock(return_value=(100, True)) with override_settings(PLUGIN_BACKEND=mock_backend): - tasks.transcode_video('videoid') + tasks.transcode_video("videoid") video_processing_state = models.ProcessingState.objects.get() - self.assertEqual(models.ProcessingState.STATUS_SUCCESS, video_processing_state.status) + self.assertEqual( + models.ProcessingState.STATUS_SUCCESS, video_processing_state.status + ) self.assertEqual("", video_processing_state.message) self.assertEqual(100, video_processing_state.progress) def test_transcode_video_restart(self): - video = factories.VideoFactory(public_id='videoid') - models.ProcessingState.objects.filter(video=video).update(status=models.ProcessingState.STATUS_RESTART) + video = factories.VideoFactory(public_id="videoid") + models.ProcessingState.objects.filter(video=video).update( + status=models.ProcessingState.STATUS_RESTART + ) - mock_backend = Mock(return_value=Mock( - start_transcoding=Mock(return_value=[]), - iter_formats=Mock(return_value=[]), - )) + mock_backend = Mock( + return_value=Mock( + start_transcoding=Mock(return_value=[]), + iter_formats=Mock(return_value=[]), + ) + ) with override_settings(PLUGIN_BACKEND=mock_backend): tasks.transcode_video_restart() - mock_backend.return_value.start_transcoding.assert_called_once_with('videoid') + mock_backend.return_value.start_transcoding.assert_called_once_with("videoid") self.assertEqual( models.ProcessingState.STATUS_SUCCESS, - models.ProcessingState.objects.get(video=video).status + models.ProcessingState.objects.get(video=video).status, ) def test_transcode_video_restart_fails(self): - video = factories.VideoFactory(public_id='videoid') - models.ProcessingState.objects.filter(video=video).update(status=models.ProcessingState.STATUS_RESTART) + video = factories.VideoFactory(public_id="videoid") + models.ProcessingState.objects.filter(video=video).update( + status=models.ProcessingState.STATUS_RESTART + ) - mock_backend = Mock(return_value=Mock( - start_transcoding=Mock(return_value=[1]), - check_progress=Mock(side_effect=exceptions.TranscodingFailed), - )) + mock_backend = Mock( + return_value=Mock( + start_transcoding=Mock(return_value=[1]), + check_progress=Mock(side_effect=exceptions.TranscodingFailed), + ) + ) with override_settings(PLUGIN_BACKEND=mock_backend): tasks.transcode_video_restart() self.assertEqual( models.ProcessingState.STATUS_FAILED, - models.ProcessingState.objects.get(video=video).status + models.ProcessingState.objects.get(video=video).status, ) mock_backend.return_value.delete_video.assert_not_called() def test_transcode_video_thumbnail_create_fails(self): - video = factories.VideoFactory(public_id='videoid') - mock_backend = Mock(return_value=Mock( - start_transcoding=Mock(return_value=[]), - iter_formats=Mock(return_value=[]), - create_thumbnail=Mock(side_effect=ValueError("description")), - )) + video = factories.VideoFactory(public_id="videoid") + mock_backend = Mock( + return_value=Mock( + start_transcoding=Mock(return_value=[]), + iter_formats=Mock(return_value=[]), + create_thumbnail=Mock(side_effect=ValueError("description")), + ) + ) with override_settings(PLUGIN_BACKEND=mock_backend): - tasks.transcode_video('videoid') + tasks.transcode_video("videoid") processing_state = models.ProcessingState.objects.get(video=video) self.assertEqual(models.ProcessingState.STATUS_FAILED, processing_state.status) self.assertEqual("thumbnail creation: description", processing_state.message) def test_video_is_deleted_during_transcoding(self): - factories.VideoFactory(public_id='videoid') + factories.VideoFactory(public_id="videoid") def start_transcoding(video_id): - models.Video.objects.filter(public_id='videoid').delete() + models.Video.objects.filter(public_id="videoid").delete() return [] - mock_backend = Mock(return_value=Mock( - start_transcoding=start_transcoding, - iter_formats=Mock(return_value=[]), - )) + mock_backend = Mock( + return_value=Mock( + start_transcoding=start_transcoding, iter_formats=Mock(return_value=[]) + ) + ) with override_settings(PLUGIN_BACKEND=mock_backend): - tasks.transcode_video('videoid') + tasks.transcode_video("videoid") self.assertEqual(0, models.Video.objects.count()) self.assertEqual(0, models.ProcessingState.objects.count()) @@ -286,7 +318,6 @@ def start_transcoding(video_id): class SubtitleTasksTest(TestCase): - def test_upload_subtitle(self): srt_content = """1 00:00:00,822 --> 00:00:01,565 @@ -313,56 +344,62 @@ def test_upload_subtitle(self): mock_backend = Mock(return_value=Mock(upload_subtitle=Mock())) with override_settings(PLUGIN_BACKEND=mock_backend): - tasks.upload_subtitle('videoid', 'subtitleid', 'fr', srt_content.encode('utf-8')) - mock_backend.return_value.upload_subtitle.assert_called_once_with('videoid', 'subtitleid', 'fr', vtt_content) + tasks.upload_subtitle( + "videoid", "subtitleid", "fr", srt_content.encode("utf-8") + ) + mock_backend.return_value.upload_subtitle.assert_called_once_with( + "videoid", "subtitleid", "fr", vtt_content + ) def test_upload_subtitle_with_invalid_format(self): mock_backend = Mock(return_value=Mock(upload_subtitle=Mock())) with override_settings(PLUGIN_BACKEND=mock_backend): self.assertRaises( exceptions.SubtitleInvalid, - tasks.upload_subtitle, 'videoid', 'subtitleid', 'fr', b'Some invalid content' + tasks.upload_subtitle, + "videoid", + "subtitleid", + "fr", + b"Some invalid content", ) -class UploadUrlsTasksTests(TestCase): +class UploadUrlsTasksTests(TestCase): def test_clean_upload_urls(self): factories.VideoUploadUrlFactory( - public_video_id='available', - expires_at=time(), - was_used=False + public_video_id="available", expires_at=time(), was_used=False ) factories.VideoUploadUrlFactory( - public_video_id='expired', - expires_at=time() - 7200, - was_used=False + public_video_id="expired", expires_at=time() - 7200, was_used=False ) factories.VideoUploadUrlFactory( - public_video_id='expired_used', - expires_at=time() - 7200, - was_used=True + public_video_id="expired_used", expires_at=time() - 7200, was_used=True ) - send_task('clean_upload_urls') + send_task("clean_upload_urls") - upload_url_ids = [url.public_video_id for url in models.VideoUploadUrl.objects.all()] + upload_url_ids = [ + url.public_video_id for url in models.VideoUploadUrl.objects.all() + ] - self.assertIn('available', upload_url_ids) - self.assertIn('expired_used', upload_url_ids) + self.assertIn("available", upload_url_ids) + self.assertIn("expired_used", upload_url_ids) self.assertEqual(2, len(upload_url_ids)) class UploadThumbnailTests(TestCase): - def test_upload_thumbnail(self): factories.VideoFactory(public_id="videoid", public_thumbnail_id="old_thumbid") - img = open(os.path.join(os.path.dirname(__file__), 'fixtures', 'elcapitan.jpg'), 'rb') + img = open( + os.path.join(os.path.dirname(__file__), "fixtures", "elcapitan.jpg"), "rb" + ) - mock_backend = Mock(return_value=Mock( - upload_thumbnail=Mock(), - delete_thumbnail=Mock(), - )) + mock_backend = Mock( + return_value=Mock(upload_thumbnail=Mock(), delete_thumbnail=Mock()) + ) with override_settings(PLUGIN_BACKEND=mock_backend): tasks.upload_thumbnail("videoid", img) mock_backend.return_value.upload_thumbnail.assert_called_once() - mock_backend.return_value.delete_thumbnail.assert_called_once_with("videoid", "old_thumbid") + mock_backend.return_value.delete_thumbnail.assert_called_once_with( + "videoid", "old_thumbid" + ) diff --git a/pipeline/tests/test_utils.py b/pipeline/tests/test_utils.py index 59dc49a..ae4e272 100644 --- a/pipeline/tests/test_utils.py +++ b/pipeline/tests/test_utils.py @@ -1,14 +1,14 @@ import os from tempfile import NamedTemporaryFile -from django.test import TestCase from PIL import Image +from django.test import TestCase + from pipeline import utils class UtilsTests(TestCase): - def test_generate_random_id(self): id1 = utils.generate_random_id(1) id2 = utils.generate_random_id(2) @@ -18,8 +18,10 @@ def test_generate_random_id(self): def test_resize_thumbnail(self): def check_size(max_size, expected_width, expected_height): - img_path = os.path.join(os.path.dirname(__file__), 'fixtures', 'elcapitan.jpg') - out_img = NamedTemporaryFile(mode='rb', suffix=".jpg") + img_path = os.path.join( + os.path.dirname(__file__), "fixtures", "elcapitan.jpg" + ) + out_img = NamedTemporaryFile(mode="rb", suffix=".jpg") utils.resize_image(img_path, out_img.name, max_size) resized_image = Image.open(out_img.name) @@ -30,8 +32,10 @@ def check_size(max_size, expected_width, expected_height): check_size(2, 1, 2) def test_make_thumbnail(self): - image = open(os.path.join(os.path.dirname(__file__), 'fixtures', 'elcapitan.jpg'), 'rb') - out_img = NamedTemporaryFile(mode='rb', suffix=".jpg") + image = open( + os.path.join(os.path.dirname(__file__), "fixtures", "elcapitan.jpg"), "rb" + ) + out_img = NamedTemporaryFile(mode="rb", suffix=".jpg") utils.make_thumbnail(image, out_img.name) resized_image = Image.open(out_img.name) diff --git a/pipeline/tests/utils.py b/pipeline/tests/utils.py index 900f1c4..341c55b 100644 --- a/pipeline/tests/utils.py +++ b/pipeline/tests/utils.py @@ -22,6 +22,7 @@ def __call__(self): class TestPluginBackend(pipeline.backend.BaseBackend): pass + def override_plugin_backend(**kwargs): """ Override a selection of methods of the plugin backend, for test purposes. diff --git a/pipeline/utils.py b/pipeline/utils.py index f0ea4de..418f48a 100644 --- a/pipeline/utils.py +++ b/pipeline/utils.py @@ -3,19 +3,22 @@ import string from tempfile import NamedTemporaryFile -from django.conf import settings from PIL import Image +from django.conf import settings + def generate_long_random_id(): return generate_random_id(20) + def generate_random_id(length=12): """ Generate a random video id of given length. """ choices = string.ascii_letters + string.digits - return ''.join([random.choice(choices) for _ in range(0, length)]) + return "".join([random.choice(choices) for _ in range(0, length)]) + def make_thumbnail(file_object, out_path): """ @@ -27,12 +30,13 @@ def make_thumbnail(file_object, out_path): """ # Copy source image to temporary file img_extension = os.path.splitext(file_object.name)[1] - src_img = NamedTemporaryFile(mode='wb', suffix=img_extension) + src_img = NamedTemporaryFile(mode="wb", suffix=img_extension) src_img.write(file_object.read()) src_img.seek(0) resize_image(src_img.name, out_path, settings.THUMBNAILS_SIZE) + def resize_image(in_path, out_path, max_size): """ Resize an image by keeping the aspect ratio such that the maximum of @@ -46,5 +50,7 @@ def resize_image(in_path, out_path, max_size): """ in_img = Image.open(in_path) ratio = max_size * 1. / max(in_img.size) - out_img = in_img.resize((round(in_img.size[0] * ratio), round(in_img.size[1] * ratio))) + out_img = in_img.resize( + (round(in_img.size[0] * ratio), round(in_img.size[1] * ratio)) + ) out_img.save(out_path) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..cf60bc7 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,18 @@ +[isort] +combine_as_imports=1 +default_section=THIRDPARTY +force_sort_within_sections=1 +from_first=1 +include_trailing_comma=1 +indent=' ' +known_thirdparty_python=celery,boto3,botocore,pycaption, PIL +known_django=django +known_drf=rest_framework,rest_framework_swagger +known_first_party=api,contrib,pipeline,transcoding,videofront +line_length=88 +lines_after_imports=2 +multi_line_output=3 +not_skip = __init__.py +sections=FUTURE,STDLIB,THIRDPARTY_PYTHON,DJANGO,DRF,THIRDPARTY,FIRSTPARTY,LOCALFOLDER +use_parentheses=1 + diff --git a/transcoding/backend_extra.py b/transcoding/backend_extra.py index 575a8c0..6a726e1 100644 --- a/transcoding/backend_extra.py +++ b/transcoding/backend_extra.py @@ -1,15 +1,17 @@ from botocore.exceptions import ClientError + from django.conf import settings from contrib.plugins.aws.backend import Backend as AwsBackend class AwsExtraBackend(AwsBackend): - ''' + """ Extends the AWS backend, adding ability to apply a new transcoding foramt on top of the existing video formats that were initially transcoded. - ''' + """ + def apply_new_transcoding(self, public_video_id): pipeline_id = settings.ELASTIC_TRANSCODER_PIPELINE_ID src_file_key = self.get_src_file_key(public_video_id) @@ -20,15 +22,13 @@ def apply_new_transcoding(self, public_video_id): output = { # Note that the transcoded video should have public-read # permissions or be accessible by cloudfront - 'Key': self.get_video_key(public_video_id, resolution), - 'PresetId': preset_id + "Key": self.get_video_key(public_video_id, resolution), + "PresetId": preset_id, } job = self.elastictranscoder_client.create_job( - PipelineId=pipeline_id, - Input={'Key': src_file_key}, - Output=output + PipelineId=pipeline_id, Input={"Key": src_file_key}, Output=output ) - jobs.append(job['Job']) + jobs.append(job["Job"]) return jobs def iter_new_formats(self, public_video_id): @@ -36,9 +36,8 @@ def iter_new_formats(self, public_video_id): try: self.s3_client.head_object( Bucket=settings.S3_BUCKET, - Key=self.get_video_key(public_video_id, resolution) + Key=self.get_video_key(public_video_id, resolution), ) except ClientError: continue yield resolution, bitrate - diff --git a/transcoding/tasks_extra.py b/transcoding/tasks_extra.py index 9045d4b..5eaaf49 100644 --- a/transcoding/tasks_extra.py +++ b/transcoding/tasks_extra.py @@ -1,4 +1,4 @@ -''' +""" These functions miror what's done on pipeline.tasks. The difference is that we don't run the initial transciding, but instead, we are adding an extra video format. @@ -7,10 +7,9 @@ - We trigger an extra transcoding and add support for it. - There is no need to generate thumbnail, since it's supposed to be done during the initial transcoding. -''' +""" from pipeline.tasks import * - from transcoding.backend_extra import AwsExtraBackend @@ -19,7 +18,7 @@ def apply_new_transcoding(public_video_id): Args: public_video_id (str) """ - with Lock('TASK_LOCK_TRANSCODE_VIDEO:' + public_video_id, 3600) as lock: + with Lock("TASK_LOCK_TRANSCODE_VIDEO:" + public_video_id, 3600) as lock: if lock.is_acquired: try: models.invalidate_cache(public_video_id) @@ -29,10 +28,7 @@ def apply_new_transcoding(public_video_id): message = "\n".join([str(arg) for arg in e.args]) models.ProcessingState.objects.filter( video__public_id=public_video_id - ).update( - status=models.ProcessingState.STATUS_FAILED, - message=message, - ) + ).update(status=models.ProcessingState.STATUS_FAILED, message=message) raise finally: models.invalidate_cache(public_video_id) @@ -43,11 +39,11 @@ def _apply_new_transcoding(public_video_id): This function is not thread-safe. It should only be called by the transcode_video task. """ video = models.Video.objects.get(public_id=public_video_id) - processing_state = models.ProcessingState.objects.filter(video__public_id=public_video_id) + processing_state = models.ProcessingState.objects.filter( + video__public_id=public_video_id + ) processing_state.update( - progress=0, - status=models.ProcessingState.STATUS_PENDING, - started_at=now() + progress=0, status=models.ProcessingState.STATUS_PENDING, started_at=now() ) jobs = AwsExtraBackend().apply_new_transcoding(public_video_id) @@ -57,9 +53,14 @@ def _apply_new_transcoding(public_video_id): jobs_progress = [0] * len(jobs) while len(success_job_indexes) + len(error_job_indexes) < len(jobs): for job_index, job in enumerate(jobs): - if job_index not in success_job_indexes and job_index not in error_job_indexes: + if ( + job_index not in success_job_indexes + and job_index not in error_job_indexes + ): try: - jobs_progress[job_index], finished = AwsExtraBackend().check_progress(job) + jobs_progress[ + job_index + ], finished = AwsExtraBackend().check_progress(job) if finished: success_job_indexes.append(job_index) except exceptions.TranscodingFailed as e: @@ -72,7 +73,7 @@ def _apply_new_transcoding(public_video_id): # the transcoding process. processing_state.update( progress=sum(jobs_progress) * 1. / len(jobs), - status=models.ProcessingState.STATUS_PROCESSING + status=models.ProcessingState.STATUS_PROCESSING, ) # Check status @@ -83,7 +84,8 @@ def _apply_new_transcoding(public_video_id): # Create video formats first so that they are available as soon as the # video object becomes available from the API for format_name, bitrate in AwsExtraBackend().iter_new_formats(public_video_id): - models.VideoFormat.objects.create(video=video, name=format_name, bitrate=bitrate) + models.VideoFormat.objects.create( + video=video, name=format_name, bitrate=bitrate + ) processing_state.update(status=models.ProcessingState.STATUS_SUCCESS) - diff --git a/transcoding/transcode.py b/transcoding/transcode.py index 279f21a..4ee1b74 100644 --- a/transcoding/transcode.py +++ b/transcoding/transcode.py @@ -9,13 +9,13 @@ TRANSCODE_COST_PER_MIN_AUDIO = 0.00522 -logger = logging.getLogger('video-transcoding') +logger = logging.getLogger("video-transcoding") logger.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) ch.setFormatter(formatter) -fh = logging.FileHandler('/var/tmp/video-transcode.log') +fh = logging.FileHandler("/var/tmp/video-transcode.log") fh.setFormatter(formatter) logger.addHandler(fh) logger.addHandler(ch) @@ -23,15 +23,15 @@ def get_videos_to_be_transcoded(course_key): logger.info("Trying to retreive playlist for course key '{}'".format(course_key)) - # For some reasons, some courses are mapped to multiple + #  For some reasons, some courses are mapped to multiple # playlist. That's why, we are using filter() and not get(). playlist_list = Playlist.objects.filter(name=course_key) logger.info("Processing course '{}'".format(course_key)) to_be_transcoded = [] for playlist in playlist_list: - videos_queryset = playlist.videos.exclude( - formats__name='UL', - ).exclude(processing_state__status='failed') + videos_queryset = playlist.videos.exclude(formats__name="UL").exclude( + processing_state__status="failed" + ) to_be_transcoded.extend(list(videos_queryset)) return to_be_transcoded @@ -40,16 +40,20 @@ def estimate_cost(course_key): duration_list = [] for video in get_videos_to_be_transcoded(course_key): try: - sd_url = video.formats.get(name='LD').url + sd_url = video.formats.get(name="LD").url except VideoFormat.DoesNotExist: logger.warning(" Could not find URL for video '{}'".format(video)) video_duration = 0 sd_url = None if sd_url: - cmd = 'ffprobe -i {url} -show_entries format=duration -v quiet -of csv="p=0"'.format(url=sd_url) + cmd = 'ffprobe -i {url} -show_entries format=duration -v quiet -of csv="p=0"'.format( + url=sd_url + ) cmd_out = subprocess.check_output(cmd, shell=True) video_duration = float(cmd_out) - logger.info(" Duration for video {} is : {}".format(video, video_duration)) + logger.info( + " Duration for video {} is : {}".format(video, video_duration) + ) duration_list.append(video_duration) duration_sec = sum(duration_list) duration = duration_sec / 60 @@ -58,23 +62,33 @@ def estimate_cost(course_key): total_cost = cost_video + cost_audio logger.info("Found videos durations: {}".format(duration_list)) logger.info("Total duration: {} s = {} min".format(duration_sec, duration)) - logger.info("Transcode video cost for course '{}': {} USD".format(course_key, cost_video)) - logger.info("Transcode audio cost for course '{}': {} USD".format(course_key, cost_audio)) - logger.info("#### Total transcode cost for course '{}': {} USD".format(course_key, total_cost)) + logger.info( + "Transcode video cost for course '{}': {} USD".format(course_key, cost_video) + ) + logger.info( + "Transcode audio cost for course '{}': {} USD".format(course_key, cost_audio) + ) + logger.info( + "#### Total transcode cost for course '{}': {} USD".format( + course_key, total_cost + ) + ) return total_cost def transcode_video(course_key): for video in get_videos_to_be_transcoded(course_key): - logger.info(" Applying new transcoding to video '{}'".format(video.public_id)) + logger.info( + " Applying new transcoding to video '{}'".format(video.public_id) + ) apply_new_transcoding(video.public_id) def transcode_for_courses(course_key_list): - ''' + """ Run video transcode for a list of courses. Takes a list of course keys separated by spaces. - ''' + """ course_keys = course_key_list.split() cost_for_all_courses = [] for course_key in course_keys: @@ -83,7 +97,7 @@ def transcode_for_courses(course_key_list): total_cost = sum(cost_for_all_courses) logger.info("#### Cost for all the courses {} USD".format(total_cost)) response = input("Type 'Yes/Y' to continue: ") - if response.lower() not in ['yes', 'y']: + if response.lower() not in ["yes", "y"]: return for course_key in course_keys: transcode_video(course_key) diff --git a/videofront/__init__.py b/videofront/__init__.py index 6ffc434..4e02384 100644 --- a/videofront/__init__.py +++ b/videofront/__init__.py @@ -1,3 +1,3 @@ # This will make sure the app is always imported when # Django starts so that shared_task will use this app. -from .celery_videofront import app as celery_app # pylint: disable=unused-import +from .celery_videofront import app as celery_app # pylint: disable=unused-import diff --git a/videofront/celery_videofront.py b/videofront/celery_videofront.py index a178018..7f13994 100644 --- a/videofront/celery_videofront.py +++ b/videofront/celery_videofront.py @@ -2,14 +2,15 @@ from celery import Celery + # set the default Django settings module for the 'celery' program. -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'videofront.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "videofront.settings") # pylint: disable=wrong-import-position -from django.conf import settings +from django.conf import settings # isort:skip -app = Celery('videofront') -app.config_from_object('django.conf:settings') +app = Celery("videofront") +app.config_from_object("django.conf:settings") # Load automatically all tasks from all installed apps. Note that in order to @@ -25,7 +26,9 @@ def send_task(name, args=None, kwargs=None, **opts): consequence, it works only for registered tasks. """ if settings.CELERY_ALWAYS_EAGER: - task = app.tasks[name] # Raises a NotRegistered exception for unregistered tasks + task = app.tasks[ + name + ] # Raises a NotRegistered exception for unregistered tasks return task.apply(args=args, kwargs=kwargs, **opts) else: return app.send_task(name, args=args, kwargs=kwargs) diff --git a/videofront/settings.py b/videofront/settings.py index d0ce671..ecb2180 100644 --- a/videofront/settings.py +++ b/videofront/settings.py @@ -1,85 +1,84 @@ -import os from datetime import timedelta +import os from django.conf import global_settings from django.utils.translation import ugettext_lazy + # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Settings to override in production -SECRET_KEY = 'CHANGEME' +SECRET_KEY = "CHANGEME" DEBUG = True ALLOWED_HOSTS = [] # Application definition INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", # 3rd-party - 'djcelery', - 'rest_framework', - 'rest_framework.authtoken', - 'rest_framework_swagger', - + "djcelery", + "rest_framework", + "rest_framework.authtoken", + "rest_framework_swagger", # Local apps - 'api', - 'pipeline', + "api", + "pipeline", ] MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", ] -ROOT_URLCONF = 'videofront.urls' +ROOT_URLCONF = "videofront.urls" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - ], + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ] }, - }, + } ] -WSGI_APPLICATION = 'videofront.wsgi.application' +WSGI_APPLICATION = "videofront.wsgi.application" # Database # https://docs.djangoproject.com/en/1.9/ref/settings/#databases DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": os.path.join(BASE_DIR, "db.sqlite3"), } } # Caching # https://docs.djangoproject.com/en/1.9/topics/cache/#database-caching CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', - 'LOCATION': 'videofront_default_cache', + "default": { + "BACKEND": "django.core.cache.backends.db.DatabaseCache", + "LOCATION": "videofront_default_cache", } } @@ -89,25 +88,19 @@ AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator" }, + {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"}, + {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator"}, + {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, ] # Internationalization # https://docs.djangoproject.com/en/1.9/topics/i18n/ -LANGUAGE_CODE = 'en-us' -TIME_ZONE = 'UTC' +LANGUAGE_CODE = "en-us" +TIME_ZONE = "UTC" USE_I18N = True USE_L10N = True USE_TZ = True @@ -115,101 +108,80 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.9/howto/static-files/ -STATIC_URL = '/static/' +STATIC_URL = "/static/" # REST Framework REST_FRAMEWORK = { - 'DEFAULT_AUTHENTICATION_CLASSES': ( + "DEFAULT_AUTHENTICATION_CLASSES": ( # It is important to have BasicAuthentication as the first auth class # in order to prompt the user for a login/password from the GUI. - 'rest_framework.authentication.BasicAuthentication', - 'rest_framework.authentication.SessionAuthentication', - 'rest_framework.authentication.TokenAuthentication', - ), + "rest_framework.authentication.BasicAuthentication", + "rest_framework.authentication.SessionAuthentication", + "rest_framework.authentication.TokenAuthentication", + ) } # Logging LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'complete': { - 'format': '%(levelname)s %(asctime)s %(name)s %(message)s' - }, - 'simple': { - 'format': '%(levelname)s %(message)s' - }, + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "complete": {"format": "%(levelname)s %(asctime)s %(name)s %(message)s"}, + "simple": {"format": "%(levelname)s %(message)s"}, }, - 'filters': { - 'require_debug_false': { - '()': 'django.utils.log.RequireDebugFalse', - }, - 'require_debug_true': { - '()': 'django.utils.log.RequireDebugTrue', - }, + "filters": { + "require_debug_false": {"()": "django.utils.log.RequireDebugFalse"}, + "require_debug_true": {"()": "django.utils.log.RequireDebugTrue"}, }, - 'handlers': { - 'null': { - 'level': 'DEBUG', - 'class': 'logging.NullHandler', - }, - 'console': { - 'level': 'INFO', - 'filters': ['require_debug_true'], - 'class': 'logging.StreamHandler', - 'formatter': 'complete', + "handlers": { + "null": {"level": "DEBUG", "class": "logging.NullHandler"}, + "console": { + "level": "INFO", + "filters": ["require_debug_true"], + "class": "logging.StreamHandler", + "formatter": "complete", }, }, - 'loggers': { - '': { - 'handlers': ['console'], - 'level': 'INFO', - }, - 'django': { - 'handlers': ['console'], - }, - 'django.request': { - 'handlers': ['console'], - 'level': 'ERROR', - 'propagate': False, + "loggers": { + "": {"handlers": ["console"], "level": "INFO"}, + "django": {"handlers": ["console"]}, + "django.request": { + "handlers": ["console"], + "level": "ERROR", + "propagate": False, }, - 'botocore.vendored.requests.packages.urllib3.connectionpool': { - 'handlers': ['console'], - 'level': 'ERROR', - }, - 'py.warnings': { - 'handlers': ['console'], + "botocore.vendored.requests.packages.urllib3.connectionpool": { + "handlers": ["console"], + "level": "ERROR", }, + "py.warnings": {"handlers": ["console"]}, }, } # Celery -BROKER_URL = 'amqp://guest:guest@localhost:5672//' -CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend' +BROKER_URL = "amqp://guest:guest@localhost:5672//" +CELERY_RESULT_BACKEND = "djcelery.backends.database:DatabaseBackend" CELERY_ALWAYS_EAGER = True CELERY_EAGER_PROPAGATES_EXCEPTIONS = True CELERYBEAT_SCHEDULE = { - 'clean_upload_urls': { - 'task': 'clean_upload_urls', - 'schedule': timedelta(hours=1), - }, - 'transcode_video_restart': { - 'task': 'transcode_video_restart', - 'schedule': timedelta(seconds=5), + "clean_upload_urls": {"task": "clean_upload_urls", "schedule": timedelta(hours=1)}, + "transcode_video_restart": { + "task": "transcode_video_restart", + "schedule": timedelta(seconds=5), }, } # Swagger documentation SWAGGER_SETTINGS = { - 'APIS_SORTER': 'alpha', - 'OPERATIONS_SORTER': 'alpha', - 'USE_SESSION_AUTH': False, - 'VALIDATOR_URL': None, + "APIS_SORTER": "alpha", + "OPERATIONS_SORTER": "alpha", + "USE_SESSION_AUTH": False, + "VALIDATOR_URL": None, } ############################## @@ -217,10 +189,10 @@ ############################## # Maximum size of subtitle files -SUBTITLES_MAX_BYTES = 1024*1024*5 # 5 Mb +SUBTITLES_MAX_BYTES = 1024 * 1024 * 5 # 5 Mb # Override this setting to provide your own custom implementation of pipeline tasks. -PLUGIN_BACKEND = 'pipeline.backend.BaseBackend' +PLUGIN_BACKEND = "pipeline.backend.BaseBackend" # Maximum of width and height size for video thumbnails THUMBNAILS_SIZE = 1024 diff --git a/videofront/settings_prod_sample_aws.py b/videofront/settings_prod_sample_aws.py index ec607de..ebb9966 100644 --- a/videofront/settings_prod_sample_aws.py +++ b/videofront/settings_prod_sample_aws.py @@ -1,40 +1,46 @@ import sys -from .settings import * # pylint: disable=unused-wildcard-import -if len(sys.argv) > 1 and sys.argv[1] == 'test': - sys.stderr.write("You are running tests with production settings. I'm pretty sure you don't want to do that.\n") +from .settings import * # pylint: disable=unused-wildcard-import + + +if len(sys.argv) > 1 and sys.argv[1] == "test": + sys.stderr.write( + "You are running tests with production settings. I'm pretty sure you don't want to do that.\n" + ) sys.exit(1) -SECRET_KEY = 'putsomerandomtextherehere' +SECRET_KEY = "putsomerandomtextherehere" DEBUG = False -ALLOWED_HOSTS = ['example.com'] +ALLOWED_HOSTS = ["example.com"] CELERY_ALWAYS_EAGER = False # This is only useful for storing videos on S3 -INSTALLED_APPS += ['contrib.plugins.aws'] +INSTALLED_APPS += ["contrib.plugins.aws"] PLUGIN_BACKEND = "contrib.plugins.aws.backend.Backend" -AWS_ACCESS_KEY_ID = 'awsaccesskey' -AWS_SECRET_ACCESS_KEY = 'awssecretaccesskey' -AWS_REGION = 'eu-west-1' # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions +AWS_ACCESS_KEY_ID = "awsaccesskey" +AWS_SECRET_ACCESS_KEY = "awssecretaccesskey" +AWS_REGION = ( + "eu-west-1" +) # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions # S3 bucket that will store all public video assets. -S3_BUCKET = 's3bucket' +S3_BUCKET = "s3bucket" # S3 bucket that will store all private video assets. In particular, source # video files will be stored in this bucket. If you do not wish your source # video files to be private, just set this setting to the same value as # S3_BUCKET. -S3_PRIVATE_BUCKET = 's3privatebucket' +S3_PRIVATE_BUCKET = "s3privatebucket" # Eventually use a cloudfront distribution to stream and download objects # CLOUDFRONT_DOMAIN_NAME = "xxxx.cloudfront.net" # Presets are of the form: (name, ID, bitrate) ELASTIC_TRANSCODER_PRESETS = [ - ('LD', '1351620000001-000030', 900), # System preset: Generic 480p 4:3 - ('SD', '1351620000001-000010', 2400), # System preset: Generic 720p - ('HD', '1351620000001-000001', 5400), # System preset: Generic 1080p + ("LD", "1351620000001-000030", 900), # System preset: Generic 480p 4:3 + ("SD", "1351620000001-000010", 2400), # System preset: Generic 720p + ("HD", "1351620000001-000001", 5400), # System preset: Generic 1080p ] -ELASTIC_TRANSCODER_THUMBNAILS_PRESET = '1351620000001-000001' -ELASTIC_TRANSCODER_PIPELINE_ID = 'yourpipelineid' +ELASTIC_TRANSCODER_THUMBNAILS_PRESET = "1351620000001-000001" +ELASTIC_TRANSCODER_PIPELINE_ID = "yourpipelineid" diff --git a/videofront/urls.py b/videofront/urls.py index d0c5a79..b127174 100644 --- a/videofront/urls.py +++ b/videofront/urls.py @@ -1,11 +1,10 @@ -from django.conf.urls import url, include +from django.conf.urls import include, url from django.contrib import admin from django.views.generic import RedirectView -urlpatterns = [ - url(r'^$', RedirectView.as_view(pattern_name='api:v1:api-root'), name='home'), - - url(r'^api/', include('api.urls')), - url(r'^admin/', admin.site.urls), +urlpatterns = [ + url(r"^$", RedirectView.as_view(pattern_name="api:v1:api-root"), name="home"), + url(r"^api/", include("api.urls")), + url(r"^admin/", admin.site.urls), ] diff --git a/videofront/wsgi.py b/videofront/wsgi.py index 227f05c..1f77c0e 100644 --- a/videofront/wsgi.py +++ b/videofront/wsgi.py @@ -11,6 +11,7 @@ from django.core.wsgi import get_wsgi_application + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "videofront.settings") application = get_wsgi_application()